def init_static_policy_distribution_after_trainer_init_callback(trainer):
trainer.storage_client = connect_storage_client()
logger.info("Initializing trainer manager interface")
trainer.manager_interface = LearnerManagerInterface(server_host=MANAGER_SEVER_HOST,
port=MANAGER_PORT,
worker_id=full_experiment_name,
storage_client=trainer.storage_client,
minio_bucket_name=BUCKET_NAME)
selection_probs, payoff_table, payoff_table_key = get_fp_metanash_for_latest_payoff_table(
manager_interface=trainer.manager_interface,
fp_iters=METANASH_FICTITIOUS_PLAY_ITERS,
accepted_opponent_policy_class_names=ACCEPTED_OPPONENT_POLICY_CLASS_NAMES,
accepted_opponent_model_config_keys=ACCEPTED_OPPONENT_MODEL_CONFIG_KEYS,
add_payoff_matrix_noise_std_dev=0.0,
mix_with_uniform_dist_coeff=PSRO_EXPLORATION_COEFF
)
if selection_probs is None:
assert payoff_table is None
assert payoff_table_key is None
print("Payoff table is empty so using random weights for static policy.")
else:
print(f"Payoff table loaded from {payoff_table_key}")
print(f"Policy selection probs: {selection_probs}")
payoff_table_dill_str = dill.dumps(payoff_table)
def worker_set_static_policy_distribution(worker):
worker.policy_map[STATIC_POLICY].static_policy_selection_probs = selection_probs
worker.policy_map[STATIC_POLICY].payoff_table = dill.loads(payoff_table_dill_str)
worker.policy_map[STATIC_POLICY].current_policy_key = None
trainer.workers.foreach_worker(worker_set_static_policy_distribution)
def init_static_policy_distribution_after_trainer_init_callback(
trainer):
trainer.storage_client = connect_storage_client()
logger.info("Initializing trainer manager interface")
trainer.manager_interface = LearnerManagerInterface(
server_host=MANAGER_SEVER_HOST,
port=MANAGER_PORT,
worker_id=full_experiment_name,
storage_client=trainer.storage_client,
minio_bucket_name=BUCKET_NAME)
logger.info("Initializing trainer lock server interface")
trainer.lock_server_interface = LockServerInterface(
server_host=LOCK_SERVER_HOST,
port=LOCK_SERVER_PORT,
worker_id=full_experiment_name)
orig_selection_probs, payoff_table, payoff_table_key = get_fp_metanash_for_latest_payoff_table(
manager_interface=trainer.manager_interface,
fp_iters=METANASH_FICTITIOUS_PLAY_ITERS,
accepted_opponent_policy_class_names=
ACCEPTED_OPPONENT_POLICY_CLASS_NAMES,
accepted_opponent_model_config_keys=
ACCEPTED_OPPONENT_MODEL_CONFIG_KEYS,
add_payoff_matrix_noise_std_dev=0.0,
mix_with_uniform_dist_coeff=PSRO_EXPLORATION_COEFF)
if orig_selection_probs is None:
assert payoff_table is None
assert payoff_table_key is None
selection_probs = None
print(
"Payoff table is empty so using random weights for static policy."
)
else:
print(f"Payoff table loaded from {payoff_table_key}")
print(f"Original Selection Probs: {orig_selection_probs}")
policy_key_to_leave_out = get_unreserved_policy_key_with_priorities(
lock_server_interface=trainer.lock_server_interface,
policy_keys=payoff_table.get_ordered_keys_in_payoff_matrix(
),
policy_priorities=orig_selection_probs)
if policy_key_to_leave_out is None:
selection_probs = orig_selection_probs
print(
"No policy keys available to reserve so using unaltered selection probs"
)
else:
chosen_policy_selection_prob = orig_selection_probs[
payoff_table.get_policy_spec_for_key(
policy_key_to_leave_out).get_payoff_matrix_index()]
print(
f"\n\nLeaving out {policy_key_to_leave_out}\n"
f"(Had selection prob of ({chosen_policy_selection_prob})\n\n"
)
selection_probs = get_fp_metanash_for_payoff_table(
payoff_table=payoff_table,
fp_iters=METANASH_FICTITIOUS_PLAY_ITERS,
accepted_opponent_policy_class_names=
ACCEPTED_OPPONENT_POLICY_CLASS_NAMES,
accepted_opponent_model_config_keys=
ACCEPTED_OPPONENT_MODEL_CONFIG_KEYS,
add_payoff_matrix_noise_std_dev=0.0,
leave_out_indexes=[
payoff_table.get_policy_spec_for_key(
policy_key_to_leave_out).
get_payoff_matrix_index()
],
mix_with_uniform_dist_coeff=PSRO_EXPLORATION_COEFF)
print(f"Subset Selection Probs: {selection_probs}")
if selection_probs is None:
assert payoff_table is None
assert payoff_table_key is None
print(
"Payoff table is empty so using random weights for static policy."
)
else:
print(f"Payoff table loaded from {payoff_table_key}")
print(f"Policy selection probs: {selection_probs}")
payoff_table_dill_str = dill.dumps(payoff_table)
def worker_set_static_policy_distribution(worker):
worker.policy_map[
STATIC_POLICY].static_policy_selection_probs = selection_probs
worker.policy_map[STATIC_POLICY].payoff_table = dill.loads(
payoff_table_dill_str)
worker.policy_map[STATIC_POLICY].current_policy_key = None
trainer.workers.foreach_worker(
worker_set_static_policy_distribution)
def measure_exploitability_of_metanashes_as_they_become_available():
logger = get_logger()
storage_client = connect_storage_client()
worker_id = f"Exploitability_Tracker_{gethostname()}_pid_{os.getpid()}_{datetime_str()}"
manager_interface = ConsoleManagerInterface(
server_host=MANAGER_SEVER_HOST,
port=MANAGER_PORT,
worker_id=worker_id,
storage_client=storage_client,
minio_bucket_name=BUCKET_NAME,
minio_local_dir=DEFAULT_LOCAL_SAVE_PATH)
logger.info(f"Started worker \'{worker_id}\'")
# If you use ray for more than just this single example fn, you'll need to move ray.init to the top of your main()
ray.init(address=os.getenv('RAY_HEAD_NODE'),
ignore_reinit_error=True,
local_mode=True)
model_config_file_path, _ = maybe_download_object(
storage_client=storage_client,
bucket_name=BUCKET_NAME,
object_name=MODEL_CONFIG_KEY,
force_download=False)
with open(model_config_file_path, 'r') as config_file:
model_config = json.load(fp=config_file)
example_env = PokerMultiAgentEnv(env_config=POKER_ENV_CONFIG)
logger.info("\n\n\n\n\n__________________________________________\n"
f"LAUNCHED FOR {POKER_GAME_VERSION}\n"
f"__________________________________________\n\n\n\n\n")
obs_space = example_env.observation_space
act_space = example_env.action_space
preprocessor = StrategoDictFlatteningPreprocessor(obs_space=obs_space)
graph = tf.Graph()
sess = tf.Session(config=tf.ConfigProto(device_count={'GPU': 0}),
graph=graph)
def fetch_logits(policy):
return {
"behaviour_logits": policy.model.last_output(),
}
_policy_cls = POLICY_CLASS.with_updates(
extra_action_fetches_fn=fetch_logits)
with graph.as_default():
with sess.as_default():
policy = _policy_cls(obs_space=preprocessor.observation_space,
action_space=act_space,
config=with_common_config({
'model':
with_base_config(
base_config=MODEL_DEFAULTS,
extra_config=model_config),
'env':
POKER_ENV,
'env_config':
POKER_ENV_CONFIG,
'custom_preprocessor':
STRATEGO_PREPROCESSOR
}))
def set_policy_weights(weights_key):
weights_file_path, _ = maybe_download_object(
storage_client=storage_client,
bucket_name=BUCKET_NAME,
object_name=weights_key,
force_download=False)
policy.load_model_weights(weights_file_path)
print("(Started Successfully)")
last_payoff_table_key = None
while True:
payoff_table, payoff_table_key = manager_interface.get_latest_payoff_table(
infinite_retry_on_error=True)
if payoff_table_key == last_payoff_table_key:
time.sleep(20)
continue
last_payoff_table_key = payoff_table_key
metanash_probs, _, _ = get_fp_metanash_for_latest_payoff_table(
manager_interface=manager_interface,
fp_iters=20000,
accepted_opponent_policy_class_names=[POLICY_CLASS_NAME],
accepted_opponent_model_config_keys=[POKER_ENV_CONFIG],
add_payoff_matrix_noise_std_dev=0.000,
mix_with_uniform_dist_coeff=None,
p_or_lower_rounds_to_zero=0.0)
if metanash_probs is not None:
policy_weights_keys = payoff_table.get_ordered_keys_in_payoff_matrix(
)
policy_dict = {
key: prob
for key, prob in zip(policy_weights_keys, metanash_probs)
}
exploitabilitly = measure_exploitability_nonlstm(
rllib_policy=policy,
poker_game_version=POKER_GAME_VERSION,
policy_mixture_dict=policy_dict,
set_policy_weights_fn=set_policy_weights)
print(f"Exploitability: {exploitabilitly}")
print(f"Starting generation {generation_index}")
active_job_list = []
active_job_policy_keys = []
if generation_index == 0:
random_policy_key = "random"
vs_random_job_str = _job_str_for_policy_key(
policy_key=random_policy_key)
assert lock_server_interface.try_to_reserve_item(
item_name=vs_random_job_str)
active_job_list.append(vs_random_job_str)
active_job_policy_keys.append(random_policy_key)
else:
# get metanash probs and make jobs for non-zero policies
selection_probs, payoff_table, payoff_table_key = get_fp_metanash_for_latest_payoff_table(
manager_interface=manager_interface,
fp_iters=20000,
add_payoff_matrix_noise_std_dev=0.0,
mix_with_uniform_dist_coeff=None)
for policy_key, prob in zip(
payoff_table.get_ordered_keys_in_payoff_matrix(),
selection_probs):
if prob > 0:
job_str = _job_str_for_policy_key(policy_key=policy_key)
assert lock_server_interface.try_to_reserve_item(
item_name=job_str)
active_job_list.append(job_str)
active_job_policy_keys.append(policy_key)
print(
f"\n\n\nLaunched the following jobs for generation {generation_index}:"
)
for job in active_job_list:
base_experiment_name = f"{CLOUD_PREFIX}learner_{POKER_GAME_VERSION}_sac_arch1_hparam_search_multexp"
full_experiment_name = f"{base_experiment_name}_{gethostname()}_pid_{os.getpid()}_{datetime_str()}"
experiment_save_dir = os.path.join(DEFAULT_RESULTS_DIR,
full_experiment_name)
manager_interface = LearnerManagerInterface(server_host=MANAGER_SEVER_HOST,
port=MANAGER_PORT,
worker_id=full_experiment_name,
storage_client=storage_client,
minio_bucket_name=BUCKET_NAME)
selection_probs, _, _ = get_fp_metanash_for_latest_payoff_table(
manager_interface=manager_interface,
fp_iters=METANASH_FICTITIOUS_PLAY_ITERS,
accepted_opponent_policy_class_names=
ACCEPTED_OPPONENT_POLICY_CLASS_NAMES,
accepted_opponent_model_config_keys=ACCEPTED_OPPONENT_MODEL_CONFIG_KEYS,
add_payoff_matrix_noise_std_dev=0.0,
mix_with_uniform_dist_coeff=PSRO_EXPLORATION_COEFF)
def init_static_policy_distribution_after_trainer_init_callback(trainer):
trainer.storage_client = connect_storage_client()
logger.info("Initializing trainer manager interface")
trainer.manager_interface = LearnerManagerInterface(
server_host=MANAGER_SEVER_HOST,
port=MANAGER_PORT,
worker_id=full_experiment_name,
storage_client=trainer.storage_client,
minio_bucket_name=BUCKET_NAME)
payoff_table, payoff_table_key = trainer.manager_interface.get_latest_payoff_table(