def get_solution_brain_set():
memory = PrioritizedMemory(
capacity=REPLAY_BUFFER_SIZE,
state_shape=(1, STATE_SIZE),
# Anneal alpha linearly
alpha_scheduler=ParameterScheduler(
initial=0.6,
lambda_fn=lambda i: 0.6 - 0.6 * i / NUM_EPISODES,
final=0.),
# Anneal beta linearly
beta_scheduler=ParameterScheduler(
initial=0.4,
final=1,
lambda_fn=lambda i: 0.4 + 0.6 * i / NUM_EPISODES
), # Anneal beta linearly
seed=SEED,
continuous_actions=True,
min_priority=MIN_PRIORITY)
reacher_brain = Brain(
brain_name=BRAIN_NAME,
action_size=ACTION_SIZE,
state_shape=STATE_SIZE,
observation_type='vector',
agents=[get_agent(memory)],
)
brain_set = BrainSet(brains=[reacher_brain])
return brain_set
def get_solution_brain_set():
params = deepcopy(default_cfg)
update_params = {
"MLP_FEATURES_HIDDEN": (512, ),
"OUTPUT_FC_HIDDEN_SIZES": (128, ),
"NUM_STACKED_FRAMES": 1,
"MLP_FEATURES_DROPOUT": None,
"OUTPUT_HIDDEN_DROPOUT": None,
"DUELING": True,
}
params.update(update_params)
policy = get_policy(ACTION_SIZE, params)
featurizer = MLP(tuple([VECTOR_STATE_SHAPE[1]] +
list(params['MLP_FEATURES_HIDDEN'])),
dropout=params['MLP_FEATURES_DROPOUT'],
activation_function=nn.ReLU(),
output_function=nn.ReLU(),
seed=SEED)
model = DQN(
VECTOR_STATE_SHAPE,
ACTION_SIZE,
featurizer,
params['MLP_FEATURES_HIDDEN'][-1],
seed=SEED,
grayscale=params["GRAYSCALE"],
num_stacked_frames=params["NUM_STACKED_FRAMES"],
output_hidden_layer_size=params["OUTPUT_FC_HIDDEN_SIZES"],
OUTPUT_HIDDEN_DROPOUT=params["OUTPUT_HIDDEN_DROPOUT"],
dueling_output=params["DUELING"],
noisy_output=params['NOISY'],
categorical_output=params['CATEGORICAL'],
)
print(model)
optimizer = torch.optim.Adam(model.parameters(), lr=params['INITIAL_LR'])
memory = get_memory(VECTOR_STATE_SHAPE, params)
solution_agent = get_agent(VECTOR_STATE_SHAPE, ACTION_SIZE, model, policy,
memory, optimizer, params)
banana_brain_ = Brain(
brain_name=BRAIN_NAME,
action_size=ACTION_SIZE,
state_shape=VECTOR_STATE_SHAPE,
observation_type='vector',
agents=[solution_agent],
)
brain_set_ = BrainSet(brains=[banana_brain_])
return brain_set_, params
def get_solution_brain_set():
agent = PPOAgent(
state_size=STATE_SIZE,
action_size=ACTION_SIZE,
seed=SEED,
actor_critic_factory=lambda: PPO_Actor_Critic(
actor_model=MLP(layer_sizes=(STATE_SIZE, 128, 128, ACTION_SIZE),
seed=SEED,
output_function=torch.nn.Tanh(),
with_batchnorm=BATCHNORM,
output_layer_initialization_fn=lambda l:
init_layer_within_range(l),
hidden_layer_initialization_fn=lambda l:
init_layer_inverse_root_fan_in(l),
activation_function=torch.nn.LeakyReLU(True),
dropout=DROPOUT),
critic_model=MLP(layer_sizes=(STATE_SIZE, 128, 128, 1),
seed=SEED,
output_function=torch.nn.Tanh(),
with_batchnorm=BATCHNORM,
output_layer_initialization_fn=lambda l:
init_layer_within_range(l),
hidden_layer_initialization_fn=lambda l:
init_layer_inverse_root_fan_in(l),
activation_function=torch.nn.LeakyReLU(True),
dropout=DROPOUT),
action_size=ACTION_SIZE,
continuous_actions=True,
),
optimizer_factory=lambda params: torch.optim.Adam(
params, lr=LR, weight_decay=WEIGHT_DECAY, eps=EPSILON),
batch_size=BATCH_SIZE,
)
crawler_brain = Brain(
brain_name=BRAIN_NAME,
action_size=ACTION_SIZE,
state_shape=STATE_SIZE,
observation_type='vector',
agents=[agent],
)
brain_set = BrainSet(brains=[crawler_brain])
return brain_set
brain_name=GOALIE_BRAIN_NAME,
action_size=GOALIE_ACTION_SIZE,
state_shape=GOALIE_STATE_SIZE,
observation_type='vector',
agents=goalie_agents,
)
striker_brain = Brain(
brain_name=STRIKER_BRAIN_NAME,
action_size=STRIKER_ACTION_SIZE,
state_shape=STRIKER_STATE_SIZE,
observation_type='vector',
agents=striker_agents,
)
brain_set = BrainSet(brains=[goalie_brain, striker_brain])
for brain_name, brain in brain_set:
for agent_num, agent in enumerate(brain.agents):
agent_id = "{}_{}".format(brain_name, agent_num)
if brain_name == 'GoalieBrain':
action_size = GOALIE_ACTION_SIZE
action_range = GOALIE_ACTION_DISCRETE_RANGE
elif brain_name == 'StrikerBrain':
action_size = STRIKER_ACTION_SIZE
action_range = STRIKER_ACTION_DISCRETE_RANGE
else:
raise ValueError('f**k')
agent.policy = IndependentMADDPGPolicy(
brain_set=brain_set,
def banana_tuning(update_params: dict):
params = deepcopy(default_cfg)
params.update(update_params)
try:
params['OUTPUT_FC_HIDDEN_SIZES'] = ast.literal_eval(
params['OUTPUT_FC_HIDDEN_SIZES'])
params['SUPPORT_RANGE'] = ast.literal_eval(params['SUPPORT_RANGE'])
params['MLP_FEATURES_HIDDEN'] = ast.literal_eval(
params['MLP_FEATURES_HIDDEN'])
policy = get_policy(ACTION_SIZE, params)
featurizer = MLP(tuple([VECTOR_STATE_SHAPE[1]] +
list(params['MLP_FEATURES_HIDDEN'])),
dropout=params['MLP_FEATURES_DROPOUT'],
activation_function=nn.ReLU(True),
output_function=nn.ReLU(True),
seed=SEED)
model = DQN(
VECTOR_STATE_SHAPE,
ACTION_SIZE,
featurizer,
params['MLP_FEATURES_HIDDEN'][-1],
seed=SEED,
grayscale=params["GRAYSCALE"],
num_stacked_frames=params["NUM_STACKED_FRAMES"],
output_hidden_layer_size=params["OUTPUT_FC_HIDDEN_SIZES"],
OUTPUT_HIDDEN_DROPOUT=params["OUTPUT_HIDDEN_DROPOUT"],
dueling_output=params["DUELING"],
noisy_output=params['NOISY'],
categorical_output=params['CATEGORICAL'],
)
print(model)
optimizer = torch.optim.Adam(model.parameters(),
lr=params['INITIAL_LR'])
memory = get_memory(VECTOR_STATE_SHAPE, params)
agent = get_agent(VECTOR_STATE_SHAPE, ACTION_SIZE, model, policy,
memory, optimizer, params)
banana_brain = Brain(
brain_name=BRAIN_NAME,
action_size=ACTION_SIZE,
state_shape=VECTOR_STATE_SHAPE,
observation_type='vector',
agents=[agent],
)
brain_set = BrainSet(brains=[banana_brain])
# Run performance evaluation
performance, info = simulator.get_agent_performance(
brain_set=brain_set,
n_train_episodes=params["N_EPISODES"],
n_eval_episodes=params["N_EVAL_EPISODES"],
max_t=params["MAX_T"],
)
info['input_params'] = params
write_tuning_data(info, performance)
global TRIAL_COUNTER
TRIAL_COUNTER += 1
print("Performance is : {}".format(performance))
return performance
except Exception as e:
print(e)
return 0
def visual_banana_tuning(update_params: dict):
params = deepcopy(default_cfg)
params.update(update_params)
try:
params['SUPPORT_RANGE'] = ast.literal_eval(params['SUPPORT_RANGE'])
params['OUTPUT_FC_HIDDEN_SIZES'] = ast.literal_eval(params['OUTPUT_FC_HIDDEN_SIZES'])
params['FILTERS'] = ast.literal_eval(params['FILTERS'])
params['KERNEL_SIZES'] = [ast.literal_eval(i) for i in ast.literal_eval(params["KERNEL_SIZES"])]
params['STRIDE_SIZES'] = [ast.literal_eval(i) for i in ast.literal_eval(params["STRIDE_SIZES"])]
policy = get_policy(ACTION_SIZE, params)
print(params)
featurizer = CNN(
image_shape=IMAGE_SHAPE,
num_stacked_frames=params["NUM_STACKED_FRAMES"],
grayscale=params["GRAYSCALE"],
filters=params["FILTERS"],
kernel_sizes=params["KERNEL_SIZES"],
stride_sizes=params["STRIDE_SIZES"],
)
model = VisualDQN(
VISUAL_STATE_SHAPE,
ACTION_SIZE,
featurizer,
featurizer.output_size,
seed=SEED,
grayscale=params["GRAYSCALE"],
num_stacked_frames=params["NUM_STACKED_FRAMES"],
output_hidden_layer_size=params["OUTPUT_FC_HIDDEN_SIZES"],
OUTPUT_HIDDEN_DROPOUT=params["OUTPUT_HIDDEN_DROPOUT"],
dueling_output=params["DUELING"],
noisy_output=params['NOISY'],
categorical_output=params['CATEGORICAL'],
)
print(model)
optimizer = torch.optim.Adam(model.parameters(), lr=params['INITIAL_LR'])
memory = get_memory(VISUAL_STATE_SHAPE, params)
agent = get_agent(VISUAL_STATE_SHAPE, ACTION_SIZE, model, policy, memory, optimizer, params)
# Run performance evaluation
banana_brain = Brain(
brain_name=BRAIN_NAME,
action_size=ACTION_SIZE,
state_shape=VISUAL_STATE_SHAPE,
observation_type='visual',
agents=[agent],
preprocess_state_fn=get_preprocess_state_fn(params)
)
brain_set = BrainSet(brains=[banana_brain])
performance, info = simulator.get_agent_performance(
brain_set=brain_set,
n_train_episodes=params["N_EPISODES"],
n_eval_episodes=params["N_EVAL_EPISODES"],
max_t=params["MAX_T"],
)
info['input_params'] = params
global TRIAL_COUNTER
TRIAL_COUNTER += 1
write_tuning_data(info, performance)
print(f"Performance is : {performance}")
return performance
except Exception as e:
# Failures can occur do to invalid CNN sizes
print("FAILURE IN HYPERPARAMETER TUNING::: {}, {}".format(e, sys.exc_info()))
return 0
def get_solution_brain_set():
tennis_agents = []
for i in range(2):
key = "TennisBrain_{}".format(i)
agent = MAPPOAgent(
agent_id=key,
state_size=STATE_SIZE,
action_size=ACTION_SIZE,
map_agent_to_state_slice={
"TennisBrain_0": lambda t: t[:, 0:24],
"TennisBrain_1": lambda t: t[:, 24:48]
},
map_agent_to_action_slice={
"TennisBrain_0": lambda t: t[:, 0:2],
"TennisBrain_1": lambda t: t[:, 2:4]
},
actor_critic_factory=lambda: MAPPO_Actor_Critic(
actor_model=MLP(
layer_sizes=(STATE_SIZE, 256, 128, ACTION_SIZE),
seed=SEED,
# output_function=BoundVectorNorm(),
output_function=torch.nn.Tanh(),
with_batchnorm=BATCHNORM,
activation_function=torch.nn.ReLU(True),
hidden_layer_initialization_fn=
init_layer_inverse_root_fan_in,
output_layer_initialization_fn=get_init_layer_within_rage(
limit_range=(-3e-4, 3e-4)),
dropout=DROPOUT),
critic_model=MACritic(
state_featurizer=MLP(
layer_sizes=(STATE_SIZE * 2 + ACTION_SIZE, 256),
with_batchnorm=BATCHNORM,
dropout=DROPOUT,
seed=SEED,
output_function=torch.nn.ReLU(),
),
output_module=MLP(
layer_sizes=(256 + ACTION_SIZE, 128, 1),
with_batchnorm=BATCHNORM,
dropout=DROPOUT,
seed=SEED,
output_layer_initialization_fn=
get_init_layer_within_rage(limit_range=(-3e-4, 3e-4)),
activation_function=torch.nn.ReLU(True),
),
),
action_size=ACTION_SIZE,
continuous_actions=True,
),
optimizer_factory=lambda params: torch.optim.AdamW(
params, lr=LR, weight_decay=WEIGHT_DECAY, eps=EPSILON),
continuous_action_range_clip=(-1, 1),
batch_size=256,
min_batches_for_training=16,
num_learning_updates=10,
beta_scheduler=ParameterScheduler(initial=0.01,
lambda_fn=lambda i: 0.01,
final=0.01),
std_scale_scheduler=ParameterScheduler(
initial=0.8, lambda_fn=lambda i: 0.8 * 0.999**i, final=0.2),
seed=SEED)
tennis_agents.append(agent)
tennis_brain = Brain(
brain_name="TennisBrain",
action_size=ACTION_SIZE,
state_shape=STATE_SIZE,
observation_type='vector',
agents=tennis_agents,
)
brain_set = BrainSet(brains=[tennis_brain])
return brain_set
def get_solution_brain_set():
# Define the solution hyper parameters
params = deepcopy(default_cfg)
update_params = {
"INITIAL_LR": 5e-4,
"NUM_STACKED_FRAMES": 4,
"OUTPUT_HIDDEN_DROPOUT": 0.1,
"DUELING": True,
"NOISY": True,
"BATCH_SIZE": 64,
"N_FILTERS": (64, 128, 128),
"EPS_DECAY_FACTOR": 0.995,
"KERNEL_SIZES": [(1, 8, 8), (1, 4, 4), (4, 3, 3)],
"STRIDE_SIZES": [(1, 4, 4), (1, 2, 2), (1, 3, 3)],
"OUTPUT_FC_HIDDEN_SIZES": (1024, ),
"WARMUP_STEPS": 10000,
}
params.update(update_params)
print("Params are: {}".format(json.dumps(params, indent=2)))
policy = get_policy(ACTION_SIZE, params)
featurizer = CNN(
image_shape=VISUAL_STATE_SHAPE[1:],
num_stacked_frames=params["NUM_STACKED_FRAMES"],
grayscale=params["GRAYSCALE"],
nfilters=params["N_FILTERS"],
kernel_sizes=params["KERNEL_SIZES"],
stride_sizes=params["STRIDE_SIZES"],
)
model = VisualDQN(
VISUAL_STATE_SHAPE,
ACTION_SIZE,
featurizer,
featurizer.output_size,
seed=SEED,
grayscale=params["GRAYSCALE"],
num_stacked_frames=params["NUM_STACKED_FRAMES"],
output_hidden_layer_size=params["OUTPUT_FC_HIDDEN_SIZES"],
OUTPUT_HIDDEN_DROPOUT=params["OUTPUT_HIDDEN_DROPOUT"],
dueling_output=params["DUELING"],
noisy_output=params['NOISY'],
categorical_output=params['CATEGORICAL'],
)
print(model)
optimizer = torch.optim.Adam(model.parameters(), lr=params['INITIAL_LR'])
memory = get_memory(VISUAL_STATE_SHAPE, params)
solution_agent = get_agent(VISUAL_STATE_SHAPE, ACTION_SIZE, model, policy,
memory, optimizer, params)
banana_brain_ = Brain(brain_name=BRAIN_NAME,
action_size=ACTION_SIZE,
state_shape=VISUAL_STATE_SHAPE,
observation_type='visual',
agents=[solution_agent],
preprocess_state_fn=get_preprocess_state_fn(params))
brain_set_ = BrainSet(brains=[banana_brain_])
return brain_set_, params
def get_solution_brain_set():
tennis_agents = []
state_featurizer = MLP(
layer_sizes=(STATE_SIZE * 2 + ACTION_SIZE, 400),
with_batchnorm=BATCHNORM,
activation_function=torch.nn.ReLU(True),
)
output_module = MLP(
layer_sizes=(400 + ACTION_SIZE, 300, 1),
with_batchnorm=BATCHNORM,
activation_function=torch.nn.ReLU(True),
output_layer_initialization_fn=get_init_layer_within_rage(
limit_range=(-3e-4, 3e-4)))
memory_factory = lambda: PrioritizedMemory(
capacity=BUFFER_SIZE,
state_shape=(1, STATE_SIZE),
alpha_scheduler=ParameterScheduler(initial=0.6,
lambda_fn=lambda i: 0.6 - 0.6 * i /
NUM_EPISODES,
final=0.),
beta_scheduler=
ParameterScheduler(initial=0.4,
final=1,
lambda_fn=lambda i: 0.4 + 0.6 * i / NUM_EPISODES
), # Anneal beta linearly
seed=SEED,
continuous_actions=True,
min_priority=1e-4)
if MATD3:
critic_factory = lambda: MATD3Critic(
critic_model_factory=lambda: MACritic(
state_featurizer=state_featurizer,
output_module=output_module,
seed=SEED,
),
seed=SEED)
else:
critic_factory = lambda: MACritic(
state_featurizer=state_featurizer,
output_module=output_module,
)
for i in range(2):
key = "TennisBrain_{}".format(i)
tennis_agent = MADDPGAgent(
key,
None,
STATE_SIZE,
ACTION_SIZE,
critic_factory=critic_factory,
actor_factory=lambda: MLP(
layer_sizes=(STATE_SIZE, 400, 300, ACTION_SIZE),
with_batchnorm=BATCHNORM,
dropout=DROPOUT,
output_function=BoundVectorNorm(),
output_layer_initialization_fn=init_layer_within_range,
hidden_layer_initialization_fn=init_layer_inverse_root_fan_in,
seed=SEED),
critic_optimizer_factory=lambda parameters: optim.Adam(
parameters, lr=CRITIC_LR, weight_decay=1.e-5),
actor_optimizer_factory=lambda parameters: optim.Adam(parameters,
lr=ACTOR_LR),
memory_factory=memory_factory,
seed=0,
batch_size=BATCH_SIZE,
homogeneous_agents=False,
)
tennis_agents.append(tennis_agent)
tennis_brain = Brain(
brain_name=BRAIN_NAME,
action_size=ACTION_SIZE,
state_shape=STATE_SIZE,
observation_type='vector',
agents=tennis_agents,
)
brain_set = BrainSet(brains=[tennis_brain])
# Update the policy with the independent MADDPG policy
# This is done so that each agent will receive the other agents'
# states/actions during training to guide actor learning.
for i, agent in enumerate(tennis_agents):
agent_id = "TennisBrain_{}".format(i)
agent.policy = IndependentMADDPGPolicy(
brain_set=brain_set,
agent_id=agent_id,
action_dim=ACTION_SIZE,
epsilon_scheduler=ParameterScheduler(initial=1,
lambda_fn=lambda i: 0.99**i,
final=0.01),
random_brain_action_factory=lambda: RandomBrainAction(
ACTION_SIZE,
1,
continuous_actions=True,
continuous_action_range=(-1, 1),
),
map_agent_to_state_slice={
"TennisBrain_0": lambda t: t[:, 0:24],
"TennisBrain_1": lambda t: t[:, 24:48]
},
map_agent_to_action_slice={
"TennisBrain_0": lambda t: t[:, 0:2],
"TennisBrain_1": lambda t: t[:, 2:4]
},
matd3=MATD3,
gaussian_noise_factory=lambda: GaussianNoise(),
continuous_actions=True,
continuous_actions_clip_range=(-1, 1))
return brain_set
def get_solution_brain_set():
params = {
'striker_actor_layer_size': (STRIKER_STATE_SIZE, 256, 256, len(range(*STRIKER_ACTION_DISCRETE_RANGE))),
'goalie_actor_layer_size': (GOALIE_STATE_SIZE, 256, 256, len(range(*GOALIE_ACTION_DISCRETE_RANGE))),
'striker_critic_state_featurizer_layer_size': (336*4 + 3, 256),
'striker_critic_output_layer_size': (256 + 1, 256, 1),
'goalie_critic_state_featurizer_layer_size': (336 * 4 + 3, 256),
'goalie_critic_output_layer_size': (256 + 1, 256, 1),
'batchnorm': True,
'actor_dropout': 0.1,
'critic_dropout': 0.2,
'lr': 5e-3,
'weight_decay': 1e-4,
'eps': 1e-6,
'num_ppo_epochs': 4,
'minimum_training_batches': 32,
'batch_size': 1024
}
goalie_agents = []
for agent_num in range(NUM_GOALIE_AGENTS):
key = 'GoalieBrain_{}'.format(agent_num)
if agent_num == 1:
goalie_agent = DummyMADDPGAgent(
GOALIE_STATE_SIZE,
len(range(*GOALIE_ACTION_DISCRETE_RANGE)),
seed=SEED,
map_agent_to_state_slice={
"GoalieBrain_0": lambda t: t[:, 0:336],
"GoalieBrain_1": lambda t: t[:, 336:672],
"StrikerBrain_0": lambda t: t[:, 672:1008],
"StrikerBrain_1": lambda t: t[:, 1008:]
},
map_agent_to_action_slice={
"GoalieBrain_0": lambda t: t[:, 0:1],
"GoalieBrain_1": lambda t: t[:, 1:2],
"StrikerBrain_0": lambda t: t[:, 2:3],
"StrikerBrain_1": lambda t: t[:, 3:4]
},
)
else:
goalie_agent = MAPPOAgent(
agent_id=key,
state_size=GOALIE_STATE_SIZE,
action_size=len(range(*GOALIE_ACTION_DISCRETE_RANGE)),
seed=SEED,
map_agent_to_state_slice={
"GoalieBrain_0": lambda t: t[:, 0:336],
"GoalieBrain_1": lambda t: t[:, 336:672],
"StrikerBrain_0": lambda t: t[:, 672:1008],
"StrikerBrain_1": lambda t: t[:, 1008:]
},
map_agent_to_action_slice={
"GoalieBrain_0": lambda t: t[:, 0:1],
"GoalieBrain_1": lambda t: t[:, 1:2],
"StrikerBrain_0": lambda t: t[:, 2:3],
"StrikerBrain_1": lambda t: t[:, 3:4]
},
actor_critic_factory=lambda: MAPPO_Actor_Critic(
actor_model=MLP(
layer_sizes=params['goalie_actor_layer_size'],
seed=SEED,
output_function=torch.nn.Softmax(),
with_batchnorm=params['batchnorm'],
activation_function=torch.nn.LeakyReLU(True),
dropout=params['actor_dropout']
),
critic_model=MACritic(
state_featurizer=MLP(
layer_sizes=params['goalie_critic_state_featurizer_layer_size'],
with_batchnorm=params['batchnorm'],
dropout=params['critic_dropout'],
seed=SEED
),
output_module=MLP(
layer_sizes=params['goalie_critic_output_layer_size'],
with_batchnorm=params['batchnorm'],
dropout=params['critic_dropout'],
seed=SEED,
),
),
action_size=GOALIE_ACTION_SIZE,
continuous_actions=False,
seed=SEED
),
min_batches_for_training=params['minimum_training_batches'],
num_learning_updates=params['num_ppo_epochs'],
optimizer_factory=lambda model_params: torch.optim.AdamW(
model_params, lr=params['lr'], weight_decay=params['weight_decay'], eps=params['eps']
),
continuous_actions=False,
batch_size=params['batch_size'],
beta_scheduler=ParameterScheduler(initial=0.01, lambda_fn=lambda i: 0.01, final=0.01),
std_scale_scheduler=ParameterScheduler(initial=0.8,
lambda_fn=lambda i: 0.8 * 0.999 ** i,
final=0.2),
)
print("Goalie is: {}".format(goalie_agent.online_actor_critic))
goalie_agents.append(goalie_agent)
striker_agents = []
for agent_num in range(NUM_STRIKER_AGENTS):
key = 'StrikerBrain_{}'.format(agent_num)
if agent_num == 1:
striker_agent = DummyMADDPGAgent(
STRIKER_STATE_SIZE,
len(range(*STRIKER_ACTION_DISCRETE_RANGE)),
SEED,
map_agent_to_state_slice={
"GoalieBrain_0": lambda t: t[:, 0:336],
"GoalieBrain_1": lambda t: t[:, 336:672],
"StrikerBrain_0": lambda t: t[:, 672:1008],
"StrikerBrain_1": lambda t: t[:, 1008:]
},
map_agent_to_action_slice={
"GoalieBrain_0": lambda t: t[:, 0:1],
"GoalieBrain_1": lambda t: t[:, 1:2],
"StrikerBrain_0": lambda t: t[:, 2:3],
"StrikerBrain_1": lambda t: t[:, 3:4]
},
)
else:
striker_agent = MAPPOAgent(
agent_id=key,
state_size=STRIKER_STATE_SIZE,
action_size=len(range(*STRIKER_ACTION_DISCRETE_RANGE)),
seed=SEED,
map_agent_to_state_slice={
"GoalieBrain_0": lambda t: t[:, 0:336],
"GoalieBrain_1": lambda t: t[:, 336:672],
"StrikerBrain_0": lambda t: t[:, 672:1008],
"StrikerBrain_1": lambda t: t[:, 1008:]
},
map_agent_to_action_slice={
"GoalieBrain_0": lambda t: t[:, 0:1],
"GoalieBrain_1": lambda t: t[:, 1:2],
"StrikerBrain_0": lambda t: t[:, 2:3],
"StrikerBrain_1": lambda t: t[:, 3:4]
},
actor_critic_factory=lambda: MAPPO_Actor_Critic(
actor_model=MLP(
layer_sizes=params['striker_actor_layer_size'],
seed=SEED,
output_function=torch.nn.Softmax(),
with_batchnorm=params['batchnorm'],
activation_function=torch.nn.LeakyReLU(True),
dropout=params['actor_dropout']
),
critic_model=MACritic(
state_featurizer=MLP(
layer_sizes=params['striker_critic_state_featurizer_layer_size'],
with_batchnorm=params['batchnorm'],
dropout=params['critic_dropout'],
seed=SEED,
),
output_module=MLP(
layer_sizes=params['striker_critic_output_layer_size'],
with_batchnorm=params['batchnorm'],
dropout=params['critic_dropout'],
seed=SEED,
),
),
action_size=STRIKER_ACTION_SIZE,
continuous_actions=False,
seed=SEED
),
optimizer_factory=lambda model_params: torch.optim.AdamW(
model_params, lr=params['lr'], weight_decay=params['weight_decay'], eps=params['eps']
),
min_batches_for_training=params['minimum_training_batches'],
num_learning_updates=params['num_ppo_epochs'],
continuous_actions=False,
batch_size=params['batch_size'],
beta_scheduler=ParameterScheduler(initial=0.01, lambda_fn=lambda i: 0.01, final=0.01),
std_scale_scheduler=ParameterScheduler(initial=0.8,
lambda_fn=lambda i: 0.8 * 0.999 ** i,
final=0.2),
)
print("Striker is: {}".format(striker_agent.online_actor_critic))
striker_agents.append(striker_agent)
goalie_brain = Brain(
brain_name=GOALIE_BRAIN_NAME,
action_size=GOALIE_ACTION_SIZE,
state_shape=GOALIE_STATE_SIZE,
observation_type='vector',
agents=goalie_agents,
)
striker_brain = Brain(
brain_name=STRIKER_BRAIN_NAME,
action_size=STRIKER_ACTION_SIZE,
state_shape=STRIKER_STATE_SIZE,
observation_type='vector',
agents=striker_agents,
)
brain_set = BrainSet(brains=[goalie_brain, striker_brain])
return brain_set