return model
def create_value_model():
inp = ks.Input((7, ))
x = inp
x = ks.layers.Dense(24, activation='tanh')(x)
x = ks.layers.Dense(12, activation='tanh')(x)
value = ks.layers.Dense(1, activation='linear')(x)
model = ks.Model(inputs=inp, outputs=value)
return model
agent_1 = aac.AdvantageActorCritic(create_policy_model(),
create_value_model(),
2,
lr=0.0001,
gamma=0.99,
entropy_factor=0.01)
agent_2 = aac.AdvantageActorCritic(create_policy_model(),
create_value_model(),
2,
lr=0.0001,
gamma=0.99,
entropy_factor=0.01)
env = multiplayer_car_env.MPCarEnv()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
episode = 0
while True:
state_1, state_2 = env.reset()
def run_policy(policy_type, episodes):
import gym
import tensorflow as tf
from algorithms import advantage_actor_critic as aac
from algorithms import simple_policy_optimization as spo
from algorithms import simple_policy_optimization_with_entropy as spowe
from algorithms import simple_policy_optimization_rnn as spornn
from algorithms import beta_advantage_actor_critic as baac
import numpy as np
ks = tf.keras
SEED = 420
tf.set_random_seed(SEED)
def create_policy_model_beta():
inp = ks.Input((8, ))
x = inp
x = ks.layers.Dense(64, activation='selu')(x)
x = ks.layers.Dense(32, activation='selu')(x)
alphas = ks.layers.Dense(2, activation='softplus')(x)
betas = ks.layers.Dense(2, activation='softplus')(x)
model = ks.Model(inputs=inp, outputs=[alphas, betas])
return model
def create_policy_model_entropy():
inp = ks.Input((8, ))
x = inp
x = ks.layers.Dense(64, activation='selu')(x)
x = ks.layers.Dense(32, activation='selu')(x)
means = ks.layers.Dense(2, activation='tanh')(x)
scales = ks.layers.Dense(2, activation='sigmoid')(x)
model = ks.Model(inputs=inp, outputs=[means, scales])
return model
def create_policy_model_no_entropy():
inp = ks.Input((8, ))
x = inp
x = ks.layers.Dense(64, activation='selu')(x)
x = ks.layers.Dense(32, activation='selu')(x)
means = ks.layers.Dense(2, activation='tanh')(x)
model = ks.Model(inputs=inp, outputs=means)
return model
def create_value_model():
inp = ks.Input((8, ))
x = inp
x = ks.layers.Dense(64, activation='selu')(x)
x = ks.layers.Dense(32, activation='selu')(x)
value = ks.layers.Dense(1, activation='linear')(x)
model = ks.Model(inputs=inp, outputs=value)
return model
def make_rnn_model():
inp = ks.Input((None, 8))
state_inp = ks.Input((32, ))
mem_out, new_rnn_state = ks.layers.GRU(
32, return_sequences=True, return_state=True)([inp, state_inp])
mem_out = ks.layers.TimeDistributed(
ks.layers.Dense(32, activation='selu'))(mem_out)
action_means = ks.layers.TimeDistributed(
ks.layers.Dense(2, activation='tanh'))(mem_out)
model = ks.models.Model(inputs=[inp, state_inp],
outputs=[action_means, new_rnn_state])
return model
if policy_type == "ac":
policy = aac.AdvantageActorCritic(create_policy_model_entropy(),
create_value_model(),
2,
entropy_factor=0.001,
gamma=0.99,
lr=0.001)
elif policy_type == "spowe":
policy = spowe.SimplePolicyOptimizerWithEntropy(
create_policy_model_entropy(),
2,
0.001,
gamma=0.99,
entropy_factor=0.01)
elif policy_type == "spornn":
initial_rnn_state = np.zeros((1, 32))
policy = spornn.SimplePolicyOptimizerRNN(make_rnn_model(),
2,
initial_rnn_state,
scale_value=0.6,
gamma=0.99,
lr=0.0001)
elif policy_type == "baac":
policy = baac.BetaAdvantageActorCritic(create_policy_model_beta(),
create_value_model(),
2,
entropy_factor=0.001,
gamma=0.99,
lr=0.001)
elif policy_type == "ppo":
policy = baac.BetaAdvantageActorCritic(create_policy_model_beta(),
create_value_model(),
2,
entropy_factor=0.001,
gamma=0.99,
ppo_eps=0.2,
lr=0.001)
else:
policy = spo.SimplePolicyOptimizer(create_policy_model_no_entropy(),
2,
0.001,
gamma=0.99,
scale_value=0.6)
scores = []
config = tf.ConfigProto(device_count={'GPU': 0})
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
env = gym.make("LunarLanderContinuous-v2")
for episode in range(episodes):
state = env.reset()
done = False
trajectory = []
score = 0
while not done:
actions = policy.get_actions(sess, state)
new_state, r, done, _ = env.step(actions)
r /= 1000
score += r
trajectory.append((state, actions, r))
state = new_state
policy.train(sess, trajectory)
scores.append(score)
episode += 1
return scores
mem_out = ks.layers.TimeDistributed(ks.layers.Dense(
8, activation='selu'))(mem_out)
action_means = ks.layers.TimeDistributed(
ks.layers.Dense(1, activation='tanh'))(mem_out)
model = ks.models.Model(inputs=[inp, state_inp],
outputs=[action_means, new_rnn_state])
return model
initial_rnn_state = np.zeros((1, 32))
agents = [
aac.AdvantageActorCritic(create_policy_model_entropy(),
create_value_model(),
1,
lr=0.001,
gamma=0.997,
entropy_factor=0.1,
value_loss_scale=0.03,
log=True,
lambd=0.95,
scale_multiplier=1.0),
aac.AdvantageActorCritic(create_policy_model_entropy(),
create_value_model(),
1,
lr=0.001,
gamma=0.997,
entropy_factor=0.1,
value_loss_scale=0.03,
lambd=0.95,
scale_multiplier=1.0),
# aac.AdvantageActorCritic(create_policy_model_entropy(), create_value_model(), 1, lr=0.0001, gamma=0.997,
# entropy_factor=1.0),
def run_policy(policy_type):
from algorithms import dummy_agent
import tensorflow as tf
from algorithms import advantage_actor_critic as aac
from algorithms import simple_policy_optimization as spo
from algorithms import simple_policy_optimization_with_entropy as spowe
from algorithms import simple_policy_optimization_rnn as spornn
import numpy as np
from algorithms import random_agent
from competition_system.player import Player
ks = tf.keras
# SEED = 420
# tf.set_random_seed(SEED)
# car_env.set_random_seed(SEED)
def create_policy_model_entropy():
inp = ks.Input((12,))
x = inp
x = ks.layers.Dense(64, activation='tanh')(x)
x = ks.layers.Dense(32, activation='tanh')(x)
means = ks.layers.Dense(2, activation='tanh')(x)
scales = ks.layers.Dense(2, activation='sigmoid')(x)
model = ks.Model(inputs=inp, outputs=[means, scales])
return model
def create_policy_model_no_entropy():
inp = ks.Input((12,))
x = inp
x = ks.layers.Dense(64, activation='tanh')(x)
x = ks.layers.Dense(32, activation='tanh')(x)
means = ks.layers.Dense(2, activation='tanh')(x)
model = ks.Model(inputs=inp, outputs=means)
return model
def create_value_model():
inp = ks.Input((12,))
x = inp
x = ks.layers.Dense(64, activation='tanh')(x)
x = ks.layers.Dense(32, activation='tanh')(x)
value = ks.layers.Dense(1, activation='linear')(x)
model = ks.Model(inputs=inp, outputs=value)
return model
def make_rnn_model():
inp = ks.Input((None, 12))
state_inp = ks.Input((32,))
mem_out, new_rnn_state = ks.layers.GRU(32, return_sequences=True, return_state=True)([inp, state_inp])
mem_out = ks.layers.TimeDistributed(ks.layers.Dense(32, activation='tanh'))(mem_out)
action_means = ks.layers.TimeDistributed(ks.layers.Dense(2, activation='tanh'))(mem_out)
model = ks.models.Model(inputs=[inp, state_inp], outputs=[action_means, new_rnn_state])
return model
if policy_type == "ac":
policy = aac.AdvantageActorCritic(create_policy_model_entropy(), create_value_model(), 2, entropy_factor=0.01,
gamma=0.997,
lr=0.0001)
elif policy_type == "spowe":
policy = spowe.SimplePolicyOptimizerWithEntropy(create_policy_model_entropy(), 2, 0.0001, gamma=0.997,
entropy_factor=0.01)
elif policy_type == "spornn":
initial_rnn_state = np.zeros((1, 32))
policy = spornn.SimplePolicyOptimizerRNN(make_rnn_model(), 2, initial_rnn_state, scale_value=0.6, gamma=0.997,
lr=0.0001)
elif policy_type == "spo":
policy = spo.SimplePolicyOptimizer(create_policy_model_no_entropy(), 2, 0.0001, gamma=0.997, scale_value=0.6)
elif policy_type == "random":
policy = random_agent.RandomAgent(2)
else:
policy = dummy_agent.DummyAgent(2)
scores = []
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
env = Player("%s" % str(policy), port=PORT)
while True:
state = env.reset()
done = False
trajectory = []
score = 0
while not done:
actions = policy.get_actions(sess, state)
new_state, r, done, _ = env.step(actions)
score += r
trajectory.append((state, actions, r))
state = new_state
policy.train(sess, trajectory)
scores.append(score)
return model
# lin_agent_1 = linear_agent_switch_wrapper.LinearAgentSwitchWrapper(
# aac.AdvantageActorCritic(create_policy_model_entropy(), create_value_model(), 2, lr=0.0001, gamma=0.97,
# entropy_factor=0.001, value_loss_scale=0.03, scale_multiplier=2.0, lambd=0.95),
# soccer_agent.SoccerAgent(2),
# 5000
# )
initial_rnn_state = np.zeros((1, 32))
agents = [
baac.BetaAdvantageActorCritic(create_policy_model_beta(), create_value_model(), 2, lr=0.0003, gamma=0.997,
entropy_factor=0.007, log=True, value_loss_scale=0.05, lambd=0.95),
aac.AdvantageActorCritic(create_policy_model_entropy(), create_value_model(), 2, lr=0.0001, gamma=0.997,
entropy_factor=0.0001, value_loss_scale=0.03, scale_multiplier=5.0, lambd=0.95),
spornn.SimplePolicyOptimizerRNN(make_rnn_model(), 2, initial_rnn_state, scale_value=3.0, gamma=0.997,
lr=0.0001),
spo.SimplePolicyOptimizer(create_policy_model_no_entropy(), 2, 0.0001, gamma=0.997, scale_value=0.6),
dummy_agent.DummyAgent(2),
random_agent.RandomAgent(2),
# soccer_agent.SoccerAgent(2),
# lin_agent_1
]
pids = {agents[i]: i for i in range(len(agents))}
pids_in_queue = set(pids.values())
def name(agent):
return str(agent.__class__.__name__) + str(id(agent))
return_sequences=True,
return_state=True)([inp, state_inp])
mem_out = ks.layers.TimeDistributed(ks.layers.Dense(
64, activation='tanh'))(mem_out)
action_means = ks.layers.TimeDistributed(
ks.layers.Dense(2, activation='tanh'))(mem_out)
model = ks.models.Model(inputs=[inp, state_inp],
outputs=[action_means, new_rnn_state])
return model
initial_rnn_state = np.zeros((1, 128))
blue_agents = [
aac.AdvantageActorCritic(create_policy_model_entropy(),
create_value_model(),
2,
lr=0.0001,
gamma=0.99,
entropy_factor=1.0),
spowe.SimplePolicyOptimizerWithEntropy(create_policy_model_entropy(),
2,
0.0001,
gamma=0.99,
entropy_factor=1.0),
spornn.SimplePolicyOptimizerRNN(make_rnn_model(),
2,
initial_rnn_state,
scale_value=0.6,
gamma=0.99,
lr=0.0001),
spo.SimplePolicyOptimizer(create_policy_model_no_entropy(),
2,