def get_final_results(simulations, psus, eps, sessions, remote_threshold):
flat_timesteps, tensor_fields = [], []
for sim_result, psu, ep in list(zip(simulations, psus, eps)):
flat_timesteps.append(flatten(sim_result))
tensor_fields.append(create_tensor_field(psu, ep))
flat_simulations = flatten(flat_timesteps)
if config_amt == 1:
return simulations, tensor_fields, sessions
elif config_amt > 1:
return flat_simulations, tensor_fields, sessions
def get_final_results(simulations, psus, eps, sessions,
remote_threshold):
flat_timesteps, tensor_fields = [], []
for sim_result, psu, ep in tqdm(list(zip(simulations, psus, eps)),
total=len(simulations),
desc='Flattening results'):
flat_timesteps.append(flatten(sim_result))
tensor_fields.append(create_tensor_field(psu, ep))
flat_simulations = flatten(flat_timesteps)
if config_amt == 1:
return simulations, tensor_fields, sessions
elif (config_amt > 1): # and (config_amt < remote_threshold):
return flat_simulations, tensor_fields, sessions
def single_proc_exec(
simulation_execs: List[Callable],
var_dict_list: List[VarDictType],
states_lists: List[StatesListsType],
configs_structs: List[ConfigsType],
env_processes_list: List[EnvProcessesType],
Ts: List[range],
SimIDs,
Ns: List[int],
ExpIDs: List[int],
SubsetIDs,
SubsetWindows,
configured_n
):
print(f'Execution Mode: single_threaded')
params = [
simulation_execs, states_lists, configs_structs, env_processes_list, Ts, SimIDs, Ns, SubsetIDs, SubsetWindows
]
simulation_exec, states_list, config, env_processes, T, sim_id, N, subset_id, subset_window = list(
map(lambda x: x.pop(), params)
)
result = simulation_exec(
var_dict_list, states_list, config, env_processes, T, sim_id, N, subset_id, subset_window, configured_n
)
return flatten(result)
def simulation(self, sweep_dict: Dict[str, List[Any]],
states_list: List[Dict[str, Any]],
configs: List[Tuple[List[Callable], List[Callable]]],
env_processes: Dict[str, Callable], time_seq: range,
runs: int) -> List[List[Dict[str, Any]]]:
def execute_run(sweep_dict, states_list, configs, env_processes,
time_seq, run) -> List[Dict[str, Any]]:
run += 1
def generate_init_sys_metrics(genesis_states_list):
for d in genesis_states_list:
d['run'], d['substep'], d['timestep'] = run, 0, 0
yield d
states_list_copy: List[Dict[str, Any]] = list(
generate_init_sys_metrics(deepcopy(states_list)))
first_timestep_per_run: List[Dict[str, Any]] = self.run_pipeline(
sweep_dict, states_list_copy, configs, env_processes, time_seq,
run)
del states_list_copy
return first_timestep_per_run
tp = TPool(runs)
pipe_run: List[List[Dict[str, Any]]] = flatten(
tp.map(
lambda run: execute_run(sweep_dict, states_list, configs,
env_processes, time_seq, run),
list(range(runs))))
tp.clear()
return pipe_run
def simulation(
self,
sweep_dict: Dict[str, List[Any]],
states_list: List[Dict[str, Any]],
configs,
env_processes: Dict[str, Callable],
time_seq: range,
simulation_id: int,
run: int,
additional_objs=None
):
def execute_run(sweep_dict, states_list, configs, env_processes, time_seq, run) -> List[Dict[str, Any]]:
run += 1
def generate_init_sys_metrics(genesis_states_list):
# for d in genesis_states_list.asDict():
for d in genesis_states_list:
d['simulation'], d['run'], d['substep'], d['timestep'] = simulation_id, run, 0, 0
yield d
states_list_copy: List[Dict[str, Any]] = list(generate_init_sys_metrics(tuple(states_list)))
first_timestep_per_run: List[Dict[str, Any]] = self.run_pipeline(
sweep_dict, states_list_copy, configs, env_processes, time_seq, run, additional_objs
)
del states_list_copy
return first_timestep_per_run
pipe_run = flatten(
[execute_run(sweep_dict, states_list, configs, env_processes, time_seq, run)]
)
return pipe_run
def parallelize_simulations(simulation_execs: List[Callable],
var_dict_list: List[VarDictType],
states_lists: List[StatesListsType],
configs_structs: List[ConfigsType],
env_processes_list: List[EnvProcessesType],
Ts: List[range], SimIDs, Ns: List[int],
ExpIDs: List[int], SubsetIDs, SubsetWindows,
configured_n):
print(f'Execution Mode: parallelized')
params = list(
zip(simulation_execs, var_dict_list, states_lists, configs_structs,
env_processes_list, Ts, SimIDs, Ns, SubsetIDs, SubsetWindows))
len_configs_structs = len(configs_structs)
unique_runs = Counter(SimIDs)
sim_count = max(unique_runs.values())
highest_divisor = int(len_configs_structs / sim_count)
new_configs_structs, new_params = [], []
for count in range(sim_count):
if count == 0:
new_params.append(params[count:highest_divisor])
new_configs_structs.append(configs_structs[count:highest_divisor])
elif count > 0:
new_params.append(params[count * highest_divisor:(count + 1) *
highest_divisor])
new_configs_structs.append(
configs_structs[count * highest_divisor:(count + 1) *
highest_divisor])
def threaded_executor(params):
tp = TPool()
if len_configs_structs > 1:
results = tp.map(
lambda t: t[0](t[1], t[2], t[3], t[4], t[5], t[6], t[7], t[8],
t[9], configured_n), params)
else:
t = params[0]
results = t[0](t[1], t[2], t[3], t[4], t[5], t[6], t[7], t[8],
t[9], configured_n)
tp.close()
return results
# pp = PPool()
# results = flatten(list(pp.map(lambda params: threaded_executor(params), new_params)))
results = flatten(
list(map(lambda params: threaded_executor(params), new_params)))
# pp.close()
# pp.join()
# pp.clear()
# pp.restart()
return results
def single_proc_exec(
simulation_execs: List[Callable],
var_dict_list: List[VarDictType],
states_lists: List[StatesListsType],
configs_structs: List[ConfigsType],
env_processes_list: List[EnvProcessesType],
Ts: List[range],
Ns: List[int]
):
l = [simulation_execs, states_lists, configs_structs, env_processes_list, Ts, Ns]
simulation_exec, states_list, config, env_processes, T, N = list(map(lambda x: x.pop(), l))
result = simulation_exec(var_dict_list, states_list, config, env_processes, T, N)
return flatten(result)
def execute(self) -> Tuple[List[Dict[str, Any]], DataFrame]:
config_proc = Processor()
create_tensor_field = TensorFieldReport(config_proc).create_tensor_field
print(r'''
__________ ____
________ __ _____/ ____/ | / __ \
/ ___/ __` / __ / / / /| | / / / /
/ /__/ /_/ / /_/ / /___/ ___ |/ /_/ /
\___/\__,_/\__,_/\____/_/ |_/_____/
by BlockScience
''')
print(f'Execution Mode: {self.exec_context + ": " + str(self.configs)}')
print(f'Configurations: {self.configs}')
var_dict_list, states_lists, Ts, Ns, eps, configs_structs, env_processes_list, partial_state_updates, simulation_execs = \
[], [], [], [], [], [], [], [], []
config_idx = 0
for x in self.configs:
Ts.append(x.sim_config['T'])
Ns.append(x.sim_config['N'])
var_dict_list.append(x.sim_config['M'])
states_lists.append([x.initial_state])
eps.append(list(x.exogenous_states.values()))
configs_structs.append(config_proc.generate_config(x.initial_state, x.partial_state_updates, eps[config_idx]))
# print(env_processes_list)
env_processes_list.append(x.env_processes)
partial_state_updates.append(x.partial_state_updates)
simulation_execs.append(SimExecutor(x.policy_ops).simulation)
config_idx += 1
final_result = None
if self.exec_context == ExecutionMode.single_proc:
tensor_field = create_tensor_field(partial_state_updates.pop(), eps.pop())
result = self.exec_method(simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns)
final_result = result, tensor_field
elif self.exec_context == ExecutionMode.multi_proc:
# if len(self.configs) > 1:
simulations = self.exec_method(simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns)
results = []
for result, partial_state_updates, ep in list(zip(simulations, partial_state_updates, eps)):
results.append((flatten(result), create_tensor_field(partial_state_updates, ep)))
final_result = results
return final_result
def simulation(
self,
sweep_dict: Dict[str, List[Any]],
states_list: List[Dict[str, Any]],
configs,
env_processes: Dict[str, Callable],
time_seq: range,
simulation_id: int,
run: int,
subset_id,
subset_window,
configured_N,
# remote_ind
additional_objs=None):
run += 1
subset_window.appendleft(subset_id)
latest_subset_id, previous_subset_id = tuple(subset_window)
if configured_N == 1 and latest_subset_id > previous_subset_id:
run -= 1
def execute_run(sweep_dict, states_list, configs, env_processes,
time_seq, _run) -> List[Dict[str, Any]]:
def generate_init_sys_metrics(genesis_states_list, sim_id,
_subset_id, _run, _subset_window):
for D in genesis_states_list:
d = D.copy()
d['simulation'], d['subset'], d['run'], d['substep'], d['timestep'] = \
sim_id, _subset_id, _run, 0, 0
yield d
states_list_copy: List[Dict[str, Any]] = list(
generate_init_sys_metrics(tuple(states_list), simulation_id,
subset_id, run, subset_window))
first_timestep_per_run: List[Dict[str, Any]] = self.run_pipeline(
sweep_dict, states_list_copy, configs, env_processes, time_seq,
run, additional_objs)
del states_list_copy
return first_timestep_per_run
pipe_run = flatten([
execute_run(sweep_dict, states_list, configs, env_processes,
time_seq, run)
])
return pipe_run
