title="Random search")
# %%
# Plot averaged cumulative rewards for DQL vs Random vs DQL-Exploit
themodel = dqla.CyberBattleStateActionModel(ep)
p.plot_averaged_cummulative_rewards(
all_runs=[best_dqn_learning_run_10, random_run, dql_exploit_run],
title=
f'Benchmark -- max_nodes={ep.maximum_node_count}, episodes={eval_episode_count},\n'
f'State: {[f.name() for f in themodel.state_space.feature_selection]} '
f'({len(themodel.state_space.feature_selection)}\n'
f"Action: abstract_action ({themodel.action_space.flat_size()})")
# %%
# plot cumulative rewards for all episodes
p.plot_all_episodes(best_dqn_learning_run_10)
##################################################
# %%
# %%
best_dqn_4 = learner.epsilon_greedy_search(
cyberbattle_gym_env=cyberbattlechain_4,
environment_properties=ep,
learner=dqla.DeepQLearnerPolicy(ep=ep,
gamma=0.15,
replay_memory_size=10000,
target_update=5,
batch_size=256,
learning_rate=0.01),
episode_count=training_episode_count,
title="Random+CredLookup"
)
# %%
randomlearning_results = learner.epsilon_greedy_search(
cyberbattlechain_10,
environment_properties=ep,
learner=CredentialCacheExploiter(),
episode_count=eval_episode_count,
iteration_count=iteration_count,
epsilon=1.0, # purely random
render=False,
verbosity=Verbosity.Quiet,
title="Random search"
)
# %%
p.plot_episodes_length([credexplot])
p.plot_all_episodes(credexplot)
all_runs = [credexplot,
randomlearning_results
]
p.plot_averaged_cummulative_rewards(
title=f'Benchmark -- max_nodes={ep.maximum_node_count}, episodes={eval_episode_count},\n',
all_runs=all_runs)
# %%
Q_source_10 = cast(a.QTabularLearner, qlearning_bestrun_10['learner']).qsource
Q_attack_10 = cast(a.QTabularLearner, qlearning_bestrun_10['learner']).qattack
p.plot_averaged_cummulative_rewards(
all_runs=all_runs,
title=f'Benchmark -- max_nodes={ep.maximum_node_count}, episodes={eval_episode_count},\n'
f'dimension={Q_source_10.state_space.flat_size()}x{Q_source_10.action_space.flat_size()}, '
f'{Q_attack_10.state_space.flat_size()}x{Q_attack_10.action_space.flat_size()}\n'
f'Q1={[f.name() for f in Q_source_10.state_space.feature_selection]} '
f'-> {[f.name() for f in Q_source_10.action_space.feature_selection]})\n'
f"Q2={[f.name() for f in Q_attack_10.state_space.feature_selection]} -> 'action'")
# %%
# plot cumulative rewards for all episodes
p.plot_all_episodes(qlearning_results[0])
# %%
# Plot the Q-matrices
# %%
# Print non-zero coordinate in the Q matrix Q_source
i = np.where(Q_source_10.qm)
q = Q_source_10.qm[i]
list(zip(np.array([Q_source_10.state_space.pretty_print(i) for i in i[0]]),
np.array([Q_source_10.action_space.pretty_print(i) for i in i[1]]), q))
# %%
# Print non-zero coordinate in the Q matrix Q_attack
i2 = np.where(Q_attack_10.qm)
# Plot averaged cumulative rewards for DQL vs Random vs DQL-Exploit
themodel = dqla.CyberBattleStateActionModel(ep)
p.plot_averaged_cummulative_rewards(
all_runs=all_runs,
title=f'Benchmark -- max_nodes={ep.maximum_node_count}, episodes={eval_episode_count},\n'
f'State: {[f.name() for f in themodel.state_space.feature_selection]} '
f'({len(themodel.state_space.feature_selection)}\n'
f"Action: abstract_action ({themodel.action_space.flat_size()})")
# %%
contenders = [
credlookup_run,
tabularq_run,
dql_run,
dql_exploit_run
]
p.plot_episodes_length(contenders)
p.plot_averaged_cummulative_rewards(
title=f'Agent Benchmark top contenders\n'
f'max_nodes:{ep.maximum_node_count}\n',
all_runs=contenders)
# %%
# Plot cumulative rewards for all episodes
for r in contenders:
p.plot_all_episodes(r)
# %%