def _simulate_in_series(env, game, nn_att, nn_def, pos, n_episodes: int, save_dir: str = None, summary_writer=None, collect_trajectories: bool=False): """ Run simulations in series on a single processor. """ attacker_rewards = np.zeros([n_episodes]) defender_rewards = np.zeros([n_episodes]) trajectories = [] for episode_i in range(n_episodes): if collect_trajectories: defender_reward, attacker_reward, traj = _run_simulation( game, nn_att, nn_def, settings.get_attacker_strategy_dir(), settings.get_defender_strategy_dir(), collect_trajectories, pos) trajectories += [traj] else: defender_reward, attacker_reward = _run_simulation( game, nn_att, nn_def, settings.get_attacker_strategy_dir(), settings.get_defender_strategy_dir(), collect_trajectories, pos) attacker_rewards[episode_i] = attacker_reward defender_rewards[episode_i] = defender_reward if collect_trajectories: return attacker_rewards, defender_rewards, trajectories else: return attacker_rewards, defender_rewards
def load_all_policies(env, str_set, opp_identity: int): """ Load all of the strategies for an agent. :param env: :param str_set: :param opp_identity: ID of the opponent (0/1 defender/attacker). :return: Dictionary from strings to `ActWrapper` policies. :rtype: dict """ if opp_identity == 0: # Pick a defender's strategy. path = settings.get_defender_strategy_dir() elif opp_identity == 1: path = settings.get_attacker_strategy_dir() else: raise ValueError("identity is neither 0 or 1!") str_dict = {} count = 1 for picked_str in str_set: # The initial policy is a function, so we do not need to load any parameters. if count == 1 and "epoch1" in picked_str: str_dict[picked_str] = fp.load_pkl(osp.join(path, picked_str)) count += 1 continue # Load the policies parameters for epoch > 1. str_dict[picked_str] = torch.load(osp.join(path, picked_str)) return str_dict
def _simulate_in_parallel(env, game, nn_att, nn_def, n_processes: int, n_episodes: int, save_dir: str = None, summary_writer=None): """ Run simulations in parallel processes. """ worker_processes = [] simulation_request_queue = multiprocessing.SimpleQueue() attacker_reward_queue = multiprocessing.SimpleQueue() defender_reward_queue = multiprocessing.SimpleQueue() # Set-up all the processes. for _ in range(n_processes): worker_processes += [SimulationWorker( simulation_request_queue, defender_reward_queue, attacker_reward_queue, nn_att, nn_def, settings.get_attacker_strategy_dir(), settings.get_defender_strategy_dir())] worker_processes[-1].start() # Request all simulations. for _ in range(n_episodes): simulation_request_queue.put(CloudpickleWrapper(game)) # Send sentinel values to tell processes to cleanly shutdown (1 per worker). for _ in range(n_processes): simulation_request_queue.put(None) for process in worker_processes: process.join() # Aggregate results. attacker_rewards = np.zeros([n_episodes]) defender_rewards = np.zeros([n_episodes]) for episode_i in range(n_episodes): attacker_rewards[episode_i] = attacker_reward_queue.get() defender_rewards[episode_i] = defender_reward_queue.get() return attacker_rewards, defender_rewards
def train(game, identity, opponent_mix_str, epoch, writer, save_path: str = None, scope: str = None): """ Train a best response policy. :param game: :param identity: :param opponent_mix_str: :param epoch: """ env = game.env env.reset_everything() env.set_training_flag(identity) if identity: # Training the attacker. if len(opponent_mix_str) != len(game.def_str): raise ValueError("The length must match while training.") env.defender.set_mix_strategy(opponent_mix_str) env.defender.set_str_set(game.def_str) if save_path is None: save_path = osp.join(settings.get_attacker_strategy_dir(), "att_str_epoch" + str(epoch) + ".pkl") else: # Training the defender. if len(opponent_mix_str) != len(game.att_str): raise ValueError("The length must match while training.") env.attacker.set_mix_strategy(opponent_mix_str) env.attacker.set_str_set(game.att_str) if save_path is None: save_path = osp.join(settings.get_defender_strategy_dir(), "def_str_epoch" + str(epoch) + ".pkl") name = "attacker" if identity else "defender" scope = name if scope is None else scope with gin.config_scope(scope): learner = learner_factory() policy, best_deviation, _, report = learner.learn_multi_nets( env, epoch=epoch, writer=writer, game=game) # add online policy, without loading policies every time. game.total_strategies[identity].append(policy) torch.save(policy, save_path, pickle_module=dill) # fp.save_pkl(replay_buffer, save_path[:-4]+".replay_buffer.pkl") return best_deviation, report
def run(load_env, env_name, n_processes): """ Run the double-oracle algorithm. """ # Create initial policies. fp.save_pkl( uniform_str_init.act_att, osp.join(settings.get_attacker_strategy_dir(), "att_str_epoch1.pkl")) fp.save_pkl( uniform_str_init.act_def, osp.join(settings.get_defender_strategy_dir(), "def_str_epoch1.pkl")) game = initialize(load_env=FLAGS.env, env_name=None, n_processes=n_processes) _run(game.env, game, meta_method_name=FLAGS.meta_method, n_processes=n_processes)
def run(self): # Because we are "spawning" the process instead of "forking" the process, we need to # reimport the run's configurations. # Reparse the flags for this process. FLAGS = flags.FLAGS FLAGS(sys.argv) # Reload gin configurations for this process. gin_files = [ osp.join(settings.SRC_DIR, "configs", f"{x}.gin") for x in FLAGS.config_files ] gin.parse_config_files_and_bindings(config_files=gin_files, bindings=FLAGS.config_overrides, skip_unknown=False) policy_name = "attacker" if self.train_attacker else "defender" for job in iter(self.job_queue.get, None): # The game we're given has no policies and has not been initialized. game, opponent = job game = game() # Unpickle game. # Register the opponent we will be playing as the opponent's only policy. if self.train_attacker: game.add_def_str(opponent) else: game.add_att_str(opponent) # The opponent sampling is done from the result directory, so we need # to copy any model we use into the policy set. if self.train_attacker: opponent_dir = settings.get_defender_strategy_dir() else: opponent_dir = settings.get_attacker_strategy_dir() new_filepath = osp.join(opponent_dir, osp.basename(opponent)) shutil.copyfile(src=opponent, dst=new_filepath) save_path = osp.join(settings.get_run_dir(), osp.basename(opponent)) save_path = save_path[:-4] # Remove ".pkl". training.train(game=game, identity=int(self.train_attacker), opponent_mix_str=np.array([1.0]), epoch=osp.basename(opponent), writer=SummaryWriter(logdir=save_path), save_path=osp.join(save_path, f"{policy_name}.pkl"))
def sample_strategy_from_mixed(env, str_set, mix_str, identity, str_dict=None): """ Sample a pure strategy from a mixed strategy. Note: str in str_set should include .pkl. :param env: :param str_set: :param mix_str: :param identity: :param str_dict: """ assert env.training_flag != identity if not len(str_set) == len(mix_str): raise ValueError( "Length of mixed strategies does not match number of strategies.") mix_str = np.array(mix_str) if np.sum(mix_str) != 1.0: mix_str = mix_str / np.sum(mix_str) picked_str = np.random.choice(str_set, p=mix_str) # TODO: modification for fast sampling. if str_dict is not None: return str_dict[picked_str], picked_str if not fp.isInName('.pkl', name=picked_str): raise ValueError('The strategy picked is not a pickle file.') if identity == 0: # pick a defender's strategy path = settings.get_defender_strategy_dir() elif identity == 1: path = settings.get_attacker_strategy_dir() else: raise ValueError("identity is neither 0 or 1!") if not fp.isExist(osp.join(path, picked_str)): raise ValueError('The strategy picked does not exist!') if "epoch1.pkl" in picked_str: act = fp.load_pkl(osp.join(path, picked_str)) return act, picked_str act = torch.load(osp.join(path, picked_str)) return act, picked_str
def _maybe_load_last_epochs_model(is_attacker: bool): """ Load the last parameters of the last trained best responder, if any. :param is_attacker: Load previous attacker's parameters. :type is_attacker: bool :return: DQNEncDec parameters, or None if no previous parameters. """ # Get the directory where all of this runs models are saved. save_dir = None filename = None if is_attacker: save_dir = settings.get_attacker_strategy_dir() filename = "att_str_epoch{}.pkl" else: save_dir = settings.get_defender_strategy_dir() filename = "def_str_epoch{}.pkl" filepath_template = osp.join(save_dir, filename) # The first epoch where we could have a checkpoint to load # is from the 2nd epoch. This is because the 1st epoch's # model are random non-parameterized functions. Therefore, # if we cannot find at least a checkpoint from the 2nd epoch, # there will be nothing to load. filepath = filepath_template.format(2) if not osp.exists(filepath): return None # Find the most recent save of a model. epoch = 2 prev_epochs_model = None while True: filepath = filepath_template.format(epoch) # If we cannot find a checkpoint for this epoch, return last checkpoint. if not osp.exists(filepath): assert prev_epochs_model is not None, "Should've been able to load at least 2nd epoch's model." return prev_epochs_model # Otherwise, update latest checkpoint. prev_epochs_model = torch.load(filepath) epoch += 1 return None
def __init__(self, env, num_episodes, threshold): # TODO: check if env should be initial env, this env should be with G_reserved. logger.info("Reminder: env in game should be same as the initial one since G should be G_reserved.") self.env = copy.deepcopy(env) self.att_str = [] self.def_str = [] self.nasheq = {} self.payoffmatrix_def = np.zeros((1, 1), dtype=np.float32) self.payoffmatrix_att = np.zeros((1, 1), dtype=np.float32) self.dir_def = settings.get_defender_strategy_dir() self.dir_att = settings.get_attacker_strategy_dir() # payoff and beneficial deviation self.att_BD_list = [] self.def_BD_list = [] self.att_payoff = [] self.def_payoff = [] self.num_episodes = num_episodes self.threshold = threshold