def train_and_show_results(self, test_spec): game_file_path = test_spec['game_file_path'] game = acpc.read_game_file(game_file_path) base_strategy, _ = read_strategy_from_file( game_file_path, test_spec['base_strategy_path']) opponent = test_spec['opponent'] opponent_strategy = create_agent_strategy_from_trained_strategy( game_file_path, base_strategy, opponent[1], opponent[2], opponent[3]) strategy, exploitability, p = RnrParameterOptimizer(game).train( opponent_strategy, test_spec['exploitability'], test_spec['max_delta']) self.assertTrue(strategy != None) self.assertTrue(is_correct_strategy(strategy)) print('Final exploitability is %s with p of %s' % (exploitability, p))
def test_kuhn_action_minus_tilted_agent(self): kuhn_equilibrium, _ = read_strategy_from_file( KUHN_POKER_GAME_FILE_PATH, 'strategies/kuhn.limit.2p-equilibrium.strategy') game = acpc.read_game_file(KUHN_POKER_GAME_FILE_PATH) exploitability = Exploitability(game) tilted_agent_strategy = create_agent_strategy_from_trained_strategy( KUHN_POKER_GAME_FILE_PATH, kuhn_equilibrium, Action.CALL, TiltType.ADD, -0.5) self.assertTrue(is_correct_strategy(tilted_agent_strategy)) self.assertTrue( not is_strategies_equal(kuhn_equilibrium, tilted_agent_strategy)) equilibrium_exploitability = exploitability.evaluate(kuhn_equilibrium) raise_add_tilted_exploitability = exploitability.evaluate( tilted_agent_strategy) self.assertTrue( raise_add_tilted_exploitability > equilibrium_exploitability)
def train_and_show_results(self, test_spec): game_file_path = test_spec['game_file_path'] game = acpc.read_game_file(game_file_path) base_strategy, _ = read_strategy_from_file( game_file_path, test_spec['base_strategy_path']) agents = test_spec['opponent_tilt_types'] num_agents = len(agents) game_name = game_file_path.split('/')[1][:-5] overwrite_figure = test_spec[ 'overwrite_figure'] if 'overwrite_figure' in test_spec else False figure_path = get_new_path( '%s/%s(it:%s-st:%s)' % (FIGURES_FOLDER, game_name, test_spec['training_iterations'], test_spec['checkpoint_iterations']), '.png', overwrite_figure) create_path_dirs(figure_path) exp = Exploitability(game) checkpoints_count = math.ceil( (test_spec['training_iterations'] - 700) / test_spec['checkpoint_iterations']) iteration_counts = np.zeros(checkpoints_count) exploitability_values = np.zeros([num_agents, checkpoints_count]) vs_opponent_utility_values = np.zeros([num_agents, checkpoints_count]) opponent_exploitability_values = np.zeros(num_agents) for i, agent in enumerate(agents): print('%s/%s' % (i + 1, num_agents)) opponent_strategy = create_agent_strategy_from_trained_strategy( game_file_path, base_strategy, agent[0], agent[1], agent[2]) self.assertTrue(is_correct_strategy(opponent_strategy)) if 'print_opponent_strategies' in test_spec and test_spec[ 'print_opponent_strategies']: write_strategy_to_file( opponent_strategy, '%s/%s.strategy' % (os.path.dirname(figure_path), get_agent_name(agent))) if 'print_best_responses' in test_spec and test_spec[ 'print_best_responses']: opponent_best_response = BestResponse(game).solve( opponent_strategy) write_strategy_to_file( opponent_best_response, '%s/%s-best_response.strategy' % (os.path.dirname(figure_path), get_agent_name(agent))) if PLOT_OPPONENT_EXPLOITABILITY: opponent_exploitability = exp.evaluate(opponent_strategy) opponent_exploitability_values[i] = opponent_exploitability print('%s exploitability: %s' % (get_agent_name(agent), opponent_exploitability)) def checkpoint_callback(game_tree, checkpoint_index, iterations): if i == 0: iteration_counts[checkpoint_index] = iterations self.assertTrue(is_correct_strategy(game_tree)) exploitability_values[i, checkpoint_index] = exp.evaluate( game_tree) vs_opponent_utility_values[i, checkpoint_index] = exp.evaluate( opponent_strategy, game_tree) rnr = RestrictedNashResponse(game, opponent_strategy, agent[3]) rnr.train(test_spec['training_iterations'], checkpoint_iterations=test_spec['checkpoint_iterations'], checkpoint_callback=checkpoint_callback) if 'print_response_strategies' in test_spec and test_spec[ 'print_response_strategies']: write_strategy_to_file( rnr.game_tree, '%s-%s-p=%s.strategy' % (figure_path[:-len('.png')], get_agent_name(agent), agent[3])) print('Vs opponent value: %s' % exp.evaluate(opponent_strategy, rnr.game_tree)) print('Exploitability: %s' % exp.evaluate(rnr.game_tree)) plt.figure(dpi=300) ax = plt.subplot(111) for j in range(i + 1): p = plt.plot(iteration_counts, exploitability_values[j], label='%s-p=%s exploitability' % (get_agent_name(agents[j]), agents[j][3]), linewidth=LINE_WIDTH) plt.plot(iteration_counts, vs_opponent_utility_values[j], '--', label='Utility against opponent strategy', color=p[0].get_color(), linewidth=LINE_WIDTH) if PLOT_OPPONENT_EXPLOITABILITY: plt.plot(iteration_counts, np.ones(checkpoints_count) * opponent_exploitability_values[j], ':', label='Opponent exploitability', color=p[0].get_color(), linewidth=LINE_WIDTH) plt.title(test_spec['title']) plt.xlabel('Training iterations') plt.ylabel('Strategy exploitability [mbb/g]') plt.grid() handles, labels = ax.get_legend_handles_labels() new_handles = [] new_labels = [] for i in range(PLOT_COUNT_PER_AGENT): for j in range(i, len(handles), PLOT_COUNT_PER_AGENT): new_handles += [handles[j]] new_labels += [labels[j]] lgd = plt.legend(new_handles, new_labels, loc='upper center', bbox_to_anchor=(0.5, -0.1), ncol=PLOT_COUNT_PER_AGENT) plt.savefig(figure_path, bbox_extra_artists=(lgd, ), bbox_inches='tight') print('Figure written to %s' % figure_path)
def run_evaluation(self, test_spec): print() workspace_dir = os.getcwd() game_file_path = workspace_dir + '/' + test_spec['game_file_path'] game = acpc.read_game_file(game_file_path) if game.get_num_players() != 2: raise AttributeError('Only games with 2 players are supported') test_name = test_spec['test_name'] base_agent = test_spec['base_agent'] validation_agents = test_spec['validation_agents'] num_matches = test_spec['num_matches'] num_match_hands = test_spec['num_match_hands'] game_name = game_file_path.split('/')[-1][:-len('.game')] validation_agent_names = [ _get_agent_name(agent) for agent in validation_agents ] test_directory = '%s/%s/%s' % (workspace_dir, FILES_PATH, test_name) agents_data_directories = [] for validation_agent in validation_agents: agent_data_dir = '%s/%s-[%s;%s]-%sx%s' % ( test_directory, game_name, base_agent[0], _get_agent_name(validation_agent), num_matches, num_match_hands) agents_data_directories += [agent_data_dir] force_recreate_data = test_spec[ 'force_recreate_data'] if 'force_recreate_data' in test_spec else False base_validation_agent_strategy = None validation_agent_strategies = [] for x in range(len(validation_agents)): agent_data_directory = agents_data_directories[x] validation_agent = validation_agents[x] data_created = True if not force_recreate_data: if os.path.exists(agent_data_directory): for i in range(num_matches): match_dir = '%s/match_%s' % (agent_data_directory, i) if not os.path.exists(match_dir) or len( os.listdir(match_dir)) == 0: data_created = False break else: data_created = False if base_validation_agent_strategy is None: base_validation_agent_strategy, _ = read_strategy_from_file( game_file_path, test_spec['base_validation_agents_strategy_path']) validation_agent_strategy = create_agent_strategy_from_trained_strategy( game_file_path, base_validation_agent_strategy, validation_agent[0], validation_agent[1], validation_agent[2]) validation_agent_strategies += [validation_agent_strategy] if not data_created or force_recreate_data: if os.path.exists(agent_data_directory): shutil.rmtree(agent_data_directory) validation_agent_strategy_path = '%s/%s.strategy' % ( test_directory, _get_agent_name(validation_agent)) write_strategy_to_file(validation_agent_strategy, validation_agent_strategy_path) for i in range(num_matches): match_data_dir = '%s/match_%s' % (agent_data_directory, i) if not os.path.exists(match_data_dir): os.makedirs(match_data_dir) seed = int(datetime.now().timestamp()) env = os.environ.copy() env['PATH'] = os.path.dirname( sys.executable) + ':' + env['PATH'] proc = subprocess.Popen([ MATCH_SCRIPT, '%s/normal' % match_data_dir, game_file_path, str(num_match_hands), str(seed), base_agent[0], _get_agent_name(validation_agent), ], cwd=ACPC_INFRASTRUCTURE_DIR, env=env, stdout=subprocess.PIPE) ports_string = proc.stdout.readline().decode( 'utf-8').strip() ports = ports_string.split(' ') args = [ (game_file_path, ports[0], base_agent[1]), (game_file_path, ports[1], validation_agent_strategy_path), ] with multiprocessing.Pool(2) as p: p.map(_run_agent, args) proc = subprocess.Popen([ MATCH_SCRIPT, '%s/reversed' % match_data_dir, game_file_path, str(num_match_hands), str(seed), _get_agent_name(validation_agent), base_agent[0], ], cwd=ACPC_INFRASTRUCTURE_DIR, env=env, stdout=subprocess.PIPE) ports_string = proc.stdout.readline().decode( 'utf-8').strip() ports = ports_string.split(' ') args = [ (game_file_path, ports[0], validation_agent_strategy_path), (game_file_path, ports[1], base_agent[1]), ] with multiprocessing.Pool(2) as p: p.map(_run_agent, args) print('Data created') output = [] def prin(string=''): nonlocal output output += [string] print(string) utility_estimators = test_spec['utility_estimators'] agents_log_files_paths = [] for x in range(len(validation_agents)): agents_data_directory = agents_data_directories[x] log_file_paths = [] for i in range(num_matches): log_file_paths += [ '%s/match_%s/normal.log' % (agents_data_directory, i), '%s/match_%s/reversed.log' % (agents_data_directory, i), ] agents_log_files_paths += [log_file_paths] agent_strategies = {} for i in range(len(validation_agents)): agent_strategies[ validation_agent_names[i]] = validation_agent_strategies[i] prin( 'Cell contains utility of row player based on observation of column player' ) for utility_estimator_spec in utility_estimators: utility_estimator_name = utility_estimator_spec[0] utility_estimator_class = utility_estimator_spec[1] utility_estimator_instance = None if utility_estimator_class is not None: if len(utility_estimator_spec) == 2: utility_estimator_instance = utility_estimator_class( game, False) elif len(utility_estimator_spec) > 2: utility_estimator_args = utility_estimator_spec[2] utility_estimator_instance = utility_estimator_class( game, False, **utility_estimator_args) prin() prin('%s (mean | SD)' % utility_estimator_name) output_table = [[None for j in range(len(validation_agents) + 1)] for i in range(len(validation_agents))] for i in range(len(validation_agents)): output_table[i][0] = validation_agent_names[i] for x in range(len(validation_agents)): log_readings = [ get_player_utilities_from_log_file( log_file_path, game_file_path=game_file_path, utility_estimator=utility_estimator_instance, player_strategies=agent_strategies, evaluated_strategies=validation_agent_strategies) for log_file_path in agents_log_files_paths[x] ] data, player_names = get_logs_data(*log_readings) means = np.mean(data, axis=0) stds = np.std(data, axis=0) player_index = player_names.index(validation_agent_names[x]) for y in range(len(validation_agents)): output_table[y][x + 1] = '%s | %s' % (means[player_index][y], stds[player_index][y]) prin( tabulate(output_table, headers=validation_agent_names, tablefmt='grid')) prin() prin('Total num hands: %s' % data.shape[0]) output_log_path = get_new_path( '%s/output-%sx%s' % (test_directory, num_matches, num_match_hands), '.log') with open(output_log_path, 'w') as file: for line in output: file.write(line + '\n')
def create_agents_and_plot_exploitabilities(self, test_spec): base_strategy, _ = read_strategy_from_file( test_spec['game_file_path'], test_spec['base_strategy_path']) game = acpc.read_game_file(test_spec['game_file_path']) exploitability = Exploitability(game) plot_equilibrium = test_spec['plot_equilibrium'] if 'plot_equilibrium' in test_spec else True if plot_equilibrium: equilibrium_exploitability = exploitability.evaluate(base_strategy) tilt_probabilities = test_spec['tilt_probabilities'] exploitability_values = np.zeros([len(TILT_TYPES), len(tilt_probabilities)]) plot_exploitabilities = test_spec['plot_exploitabilities'] if 'plot_exploitabilities' in test_spec else True if plot_exploitabilities: for i, tilt_type in enumerate(TILT_TYPES): for j, tilt_probability in enumerate(tilt_probabilities): tilted_agent = create_agent_strategy_from_trained_strategy( test_spec['game_file_path'], base_strategy, tilt_type[1], tilt_type[2], tilt_probability) exploitability_values[i, j] = exploitability.evaluate(tilted_agent) plt.figure(dpi=160) for j in range(i + 1): plt.plot( tilt_probabilities, exploitability_values[j], label=TILT_TYPES[j][0], linewidth=0.8) if plot_equilibrium: plt.plot( tilt_probabilities, [equilibrium_exploitability] * len(tilt_probabilities), 'r--', label='Equilibrium', linewidth=1.5) # plt.title(test_spec['title']) plt.xlabel('Tilt amount') plt.ylabel('Agent exploitability [mbb/g]') plt.grid() plt.legend() figure_output_path = '%s/%s.png' % (FIGURES_FOLDER, test_spec['figure_filename']) figures_directory = os.path.dirname(figure_output_path) if not os.path.exists(figures_directory): os.makedirs(figures_directory) plt.savefig(figure_output_path) plot_agent_comparison = test_spec['plot_agent_comparison'] if 'plot_agent_comparison' in test_spec else False if plot_agent_comparison: agents_strategies = [] agent_names = [] for i, tilt_type in enumerate(TILT_TYPES): for j, tilt_probability in enumerate(tilt_probabilities): agent_names += ['%s %s %s' % (str(tilt_type[1]).split('.')[1], str(tilt_type[2]).split('.')[1], tilt_probability)] agents_strategies += [create_agent_strategy_from_trained_strategy( test_spec['game_file_path'], base_strategy, tilt_type[1], tilt_type[2], tilt_probability)] num_agents = len(agent_names) scores_table = np.zeros([num_agents, num_agents]) num_comparisons = 0 for i in range(num_agents): for j in range(i, num_agents): num_comparisons += 1 with tqdm(total=num_comparisons) as pbar: for i in range(num_agents): for j in range(i, num_agents): scores_table[i, j] = exploitability.evaluate(agents_strategies[j], agents_strategies[i]) scores_table[j, i] = -scores_table[i, j] pbar.update(1) max_score = scores_table.max() min_score = scores_table.min() # plt.figure(dpi=160) fig, ax = plt.subplots() cax = plt.imshow(scores_table, cmap=plt.cm.RdYlGn) plt.xticks(np.arange(num_agents), agent_names) plt.setp(ax.xaxis.get_majorticklabels(), rotation=45, ha="right", rotation_mode="anchor") plt.yticks(np.arange(num_agents), agent_names) # plt.yticks(rotation=35) # plt.tick_params( # axis='x', # which='both', # bottom=False, # top=False, # labelbottom=False) cbar = fig.colorbar(cax, ticks=[min_score, 0, max_score]) cbar.ax.set_yticklabels([round(min_score), '0', round(max_score)]) plt.tight_layout() plt.gcf().subplots_adjust(left=0.1) figure_output_path = '%s/%s-comparison.png' % (FIGURES_FOLDER, test_spec['figure_filename']) figures_directory = os.path.dirname(figure_output_path) if not os.path.exists(figures_directory): os.makedirs(figures_directory) plt.savefig(figure_output_path, dpi=160)
def train_and_show_results(self, test_spec): game_file_path = test_spec['game_file_path'] portfolio_name = test_spec['portfolio_name'] agent_specs = test_spec['opponent_tilt_types'] if not _check_agent_names_unique(agent_specs): raise AttributeError( 'Agents must be unique so that they have unique names') strategies_directory_base = '%s/%s' % (TEST_OUTPUT_DIRECTORY, portfolio_name) strategies_directory = strategies_directory_base if 'overwrite_portfolio_path' not in test_spec or not test_spec[ 'overwrite_portfolio_path']: counter = 1 while os.path.exists(strategies_directory): strategies_directory = '%s(%s)' % (strategies_directory_base, counter) counter += 1 if not os.path.exists(strategies_directory): os.makedirs(strategies_directory) game = acpc.read_game_file(game_file_path) exp = Exploitability(game) # Delete results since they will be generated again for file in os.listdir(strategies_directory): absolute_path = '/'.join([strategies_directory, file]) if os.path.isfile(absolute_path): os.remove(absolute_path) base_strategy, _ = read_strategy_from_file( game_file_path, test_spec['base_strategy_path']) num_opponents = len(agent_specs) opponents = [] for agent in agent_specs: opponent_strategy = create_agent_strategy_from_trained_strategy( game_file_path, base_strategy, agent[0], agent[1], agent[2]) opponents += [opponent_strategy] parallel = test_spec['parallel'] if 'parallel' in test_spec else False response_paths = [ '%s/responses/%s-response.strategy' % (strategies_directory, _get_agent_name(agent)) for agent in agent_specs ] opponent_responses = [None] * num_opponents responses_to_train_indices = [] responses_to_train_opponents = [] responses_to_train_params = [] for i in range(num_opponents): if os.path.exists(response_paths[i]): response_strategy, _ = read_strategy_from_file( game_file_path, response_paths[i]) opponent_responses[i] = response_strategy else: responses_to_train_indices += [i] responses_to_train_opponents += [opponents[i]] responses_to_train_params += [agent_specs[i][3]] def on_response_trained(response_index, response_strategy): output_file_path = response_paths[ responses_to_train_indices[response_index]] output_file_dir = os.path.dirname(output_file_path) if not os.path.exists(output_file_dir): os.makedirs(output_file_dir) opponent_strategy = opponents[response_index] opponent_exploitability = exp.evaluate(opponent_strategy) response_exploitability = exp.evaluate(response_strategy) response_utility_vs_opponent = exp.evaluate( opponent_strategy, response_strategy) write_strategy_to_file(response_strategy, output_file_path, [ 'Opponent exploitability: %s' % opponent_exploitability, 'Response exploitability: %s' % response_exploitability, 'Response value vs opponent: %s' % response_utility_vs_opponent, ]) print('%s responses need to be trained' % len(responses_to_train_opponents)) responses_to_train_strategies = train_portfolio_responses( game_file_path, responses_to_train_opponents, responses_to_train_params, log=True, parallel=parallel, callback=on_response_trained) for i, j in enumerate(responses_to_train_indices): opponent_responses[j] = responses_to_train_strategies[i] if 'portfolio_cut_improvement_threshold' in test_spec: portfolio_strategies, response_indices = optimize_portfolio( game_file_path, opponents, opponent_responses, portfolio_cut_improvement_threshold=test_spec[ 'portfolio_cut_improvement_threshold'], log=True, output_directory=strategies_directory) else: portfolio_strategies, response_indices = optimize_portfolio( game_file_path, opponents, opponent_responses, log=True, output_directory=strategies_directory) portfolio_size = len(portfolio_strategies) agent_names = [ _get_agent_name(agent) for agent in np.take(agent_specs, response_indices, axis=0) ] print() for a in agent_specs: print(_get_agent_name(a)) response_strategy_file_names = [] for i, strategy in enumerate(portfolio_strategies): agent_name = agent_names[i] opponent_strategy = opponents[response_indices[i]] opponent_exploitability = exp.evaluate(opponent_strategy) response_exploitability = exp.evaluate(strategy) response_utility_vs_opponent = exp.evaluate( opponent_strategy, strategy) # Save portfolio response strategy response_strategy_output_file_path = '%s/%s-response.strategy' % ( strategies_directory, agent_name) response_strategy_file_names += [ response_strategy_output_file_path.split('/')[-1] ] write_strategy_to_file( strategy, response_strategy_output_file_path, [ 'Opponent exploitability: %s' % opponent_exploitability, 'Response exploitability: %s' % response_exploitability, 'Response value vs opponent: %s' % response_utility_vs_opponent, ]) # Save opponent strategy opponent_strategy_file_name = '%s-opponent.strategy' % agent_name opponent_strategy_output_file_path = '%s/%s' % ( strategies_directory, opponent_strategy_file_name) write_strategy_to_file(opponent_strategy, opponent_strategy_output_file_path) # Generate opponent ACPC script opponent_script_path = '%s/%s.sh' % (strategies_directory, agent_name) shutil.copy(BASE_OPPONENT_SCRIPT_PATH, opponent_script_path) _replace_in_file( opponent_script_path, OPPONENT_SCRIPT_REPLACE_STRINGS, [ WARNING_COMMENT, game_file_path, opponent_strategy_output_file_path.split('/')[-1] ]) for utility_estimation_method in UTILITY_ESTIMATION_METHODS: agent_name_method_name = '' if utility_estimation_method == UTILITY_ESTIMATION_METHODS[ 0] else '-%s' % utility_estimation_method agent_script_path = '%s/%s%s.sh' % ( strategies_directory, portfolio_name, agent_name_method_name) shutil.copy(BASE_AGENT_SCRIPT_PATH, agent_script_path) strategies_replacement = '' for i in range(portfolio_size): strategies_replacement += ' "${SCRIPT_DIR}/%s"' % response_strategy_file_names[ i] if i < (portfolio_size - 1): strategies_replacement += ' \\\n' _replace_in_file(agent_script_path, AGENT_SCRIPT_REPLACE_STRINGS, [ WARNING_COMMENT, game_file_path, '"%s"' % utility_estimation_method, strategies_replacement ])