def create(env):
np.random.seed(config.current.domain_seed)
env.seed(config.current.domain_seed)
nb_actions = env.action_space.n
# Next, we build a very simple model.
model = Sequential()
model.add(Flatten(input_shape=(1, ) + env.observation_space.shape))
model.add(Dense(config.current.agent_vfn_complexity))
model.add(Activation('relu'))
model.add(Dense(config.current.agent_vfn_complexity))
model.add(Activation('relu'))
model.add(Dense(config.current.agent_vfn_complexity))
model.add(Activation('relu'))
model.add(Dense(nb_actions))
model.add(Activation('linear'))
global graph
graph = tf.get_default_graph()
# SARSA does not require a memory.
policy = BoltzmannQPolicy()
sarsa = SARSAAgent(model=model,
nb_actions=nb_actions,
nb_steps_warmup=10,
policy=policy)
sarsa.compile(Adam(lr=1e-3), metrics=['mae'])
return sarsa
def init_sarsa(env, nb_actions, lr=1e-3):
""" Initialize the Sarsa agent using the keras-rl package.
:param env: the environment to be played, required to determine the input size
:param nb_actions: number of actions
:param lr: learning rate
:return: Sarsa Agent
"""
# Next, we build a very simple model.
model = Sequential()
model.add(Flatten(input_shape=(1, ) + env.observation_space.shape))
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dense(256))
model.add(Activation('relu'))
model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dense(nb_actions))
model.add(Activation('linear'))
# SARSA does not require a memory.
policy = BoltzmannQPolicy()
sarsa = SARSAAgent(model=model,
nb_actions=nb_actions,
nb_steps_warmup=10,
policy=policy)
sarsa.model_name = f"SARSA"
sarsa.compile(Adam(lr=lr), metrics=['mae'])
return sarsa
def run_sarsa():
global N_NODE_NETWORK
env = SnakeGymDiscrete()
nb_actions = env.action_space.n
# initialize randomness
np.random.seed(123)
env.seed(123)
# create model
model = Sequential()
model.add(Flatten(input_shape=(1, ) + env.observation_space.shape))
model.add(Dense(N_NODE_NETWORK))
model.add(Activation('relu'))
model.add(Dense(N_NODE_NETWORK))
model.add(Activation('relu'))
model.add(Dense(N_NODE_NETWORK))
model.add(Activation('relu'))
model.add(Dense(nb_actions))
model.add(Activation('linear'))
# SARSA does not require a memory.
policy = BoltzmannQPolicy()
sarsa = SARSAAgent(model=model,
nb_actions=nb_actions,
nb_steps_warmup=10,
policy=policy)
sarsa.compile(Adam(lr=1e-3), metrics=['mae'])
sarsa.fit(env, nb_steps=50000, visualize=False, verbose=2)
sarsa.save_weights('sarsa_SnakeGymDiscrete_weights.h5f', overwrite=True)
sarsa.test(env, nb_episodes=5, visualize=True)
def create_sarsa_agent(env):
env = create_environment()
model = create_deep_model(env)
nb_actions = env.action_space.n
policy = BoltzmannQPolicy()
sarsa = SARSAAgent(model=model, nb_actions=nb_actions, nb_steps_warmup=10, policy=policy)
sarsa.compile(Adam(lr=1e-3), metrics=['mae'])
return sarsa
def __init__(self, state_dim, action_space, epsilon, gamma, lr):
self._model = self._get_model(state_dim, action_space)
self.agent = SARSAAgent(self._model,
nb_actions=action_space,
gamma=gamma,
policy=EpsGreedyQPolicy(epsilon),
test_policy=EpsGreedyQPolicy(eps=0.01))
self.agent.compile(Adam(lr))
def main():
# binance = DataReader()
env = BinanceEnv()
# binance.get_recent_trades()
# env.next_observation()
# binance_market = BinanceMarket()
# binance_market.long()
# time.sleep(3)
# binance_market.close_long()
# time.sleep(3)
# binance_market.short()
# time.sleep(3)
# binance_market.close_short()
# binance_market.update_positions()
# print(binance_market.balance)
# episodes = 10
# for episode in range(1, episodes + 1):
# # At each begining reset the game
# state = env.reset()
# # set done to False
# done = False
# # set score to 0
# score = 0
# # while the game is not finished
# while not done:
# # visualize each step
# env.render()
# # choose a random action
# action = random.randint(0, 5)
# # execute the action
# n_state, reward, done, info = env.step(action)
# # keep track of rewards
# score += reward
# print('episode {} score {}'.format(episode, score))
model = agent(env.observation_space.shape[0], env.action_space.n)
policy = EpsGreedyQPolicy()
sarsa = SARSAAgent(model=model, policy=policy, nb_actions=env.action_space.n)
sarsa.compile('adam', metrics=['mse', 'accuracy'])
# sarsa.load_weights('sarsa_weights_bnb_07.h5f')
env.is_testing = False
sarsa.fit(env, nb_steps=100000, visualize=False, verbose=1)
sarsa.save_weights('sarsa_weights_bnb_07_1.h5f', overwrite=True)
# sarsa.load_weights('sarsa_weights_bnb_07_1.h5f')
# env.simulator = False
env.is_testing = True
scores = sarsa.test(env, nb_episodes=1, visualize=False)
print('Average score over 100 test games:{}'.format(np.mean(scores.history['episode_reward'])))
_ = sarsa.test(env, nb_episodes=10, visualize=True)
obs = env.reset()
for i in range(2000):
action, _states = model.predict(obs)
obs, rewards, done, info = env.step(action)
env.render()
def __init__(self, env, timesteps_per_episode=10001):
super().__init__(env, timesteps_per_episode)
self.num_episodes = 400
self.evaluating = False
self.action_size = env.action_space.n
self.state_size = env.num_states
self.model = self._build_compile_model()
self.agent = SARSAAgent(model=self.model,
nb_actions=self.action_size,
policy=EpsGreedyQPolicy())
def main():
model = Sequential()
model.add(Flatten(input_shape=(1, 7)))
model.add(Dense(units=20, activation='relu'))
model.add(Dense(units=20, activation='relu'))
model.add(Dense(units=6, activation='linear'))
logger.info(model.summary())
steps = 1E9
interval = steps // 100
# policy = MyPolicy()
policy = BoltzmannQPolicy()
agent = SARSAAgent(model=model, nb_actions=6, policy=policy, train_interval=10, nb_steps_warmup=10)
adam = Adam()
sgd = SGD(lr=1e-3, momentum=0, decay=0, nesterov=False)
agent.compile(optimizer=adam, metrics=['mse'])
env = MyEnv()
agent.fit(env, steps, verbose=2, visualize=True)
fp = Path(__file__).resolve().parent / 'sarsa_weights.h5f'
agent.save_weights(fp, overwrite=True)
logger.info('Done')
def main():
# nb_actions = cpst._action_space
nb_actions = 2
# Next, we build a very simple model.
model = Sequential()
#n_os = cpst._observation_space.shape
n_os = 4
model.add(Flatten(input_shape=[1] + [n_os]))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(nb_actions))
model.add(Activation('linear'))
print(model.summary())
model._make_predict_function()
# SARSA does not require a memory.
policy = BoltzmannQPolicy()
sarsa = SARSAAgent(model=model,
nb_actions=nb_actions,
nb_steps_warmup=10,
policy=policy)
sarsa.compile(Adam(lr=1e-3), metrics=['mae'])
cart_pole = CartPole(name='cp')
log = logging.getLogger('bact2')
RE = RunEngine({})
RE.log.setLevel('DEBUG')
cart_pole.log = RE.log
stm = [cart_pole.x, cart_pole.x_dot, cart_pole.theta, cart_pole.theta_dot]
cpst = CartPoleEnv(detectors=[cart_pole],
motors=[cart_pole],
state_motors=stm,
user_kwargs={'mode_var': cart_pole.rl_mode})
np.random.seed(123)
cpst.seed(123)
partial = functools.partial(run_test, sarsa, cpst, log=RE.log)
RE(run_environement(cpst, partial, log=RE.log))
def compile_agent(self):
# Finally, we configure and compile our agent. You can use every built-in Keras optimizer and
# even the metrics!
processor = DistopiaProcessor(self.num_blocks, self.num_actions)
#memory = SequentialMemory(limit=50000, window_length=1)
#policy = PatchedBoltzmannQPolicy(num_actions = self.num_actions, num_blocks = self.num_blocks)
#test_policy = PatchedGreedyQPolicy(num_actions = self.num_actions, num_blocks = self.num_blocks)
policy = BoltzmannQPolicy()
test_policy = GreedyQPolicy()
self.sarsa = SARSAAgent(model=self.model,
processor=processor,
nb_actions=self.nb_actions,
nb_steps_warmup=1000,
policy=policy,
test_policy=test_policy,
gamma=0.9)
self.sarsa.compile(Adam(lr=1e-3), metrics=['mae'])
def main():
# with ServerProxy("http://127.0.0.1:8000/", verbose=False, allow_none=True) as proxy:
if True:
pass
#D:\Devel\github\keras-rl;D:\Devel\github\Devel\hz-b\naus
# set PYTHONPATH=D:\Devel\github\keras-rl;D:\Devel\github\Devel\hz-b\naus
# & python d:\Devel\github\Devel\hz-b\naus\examples\rl\cart_pole\sarsa_cartpole.py
def stop_my_application():
print('Stopping application')
with allow_interrupt():
# main polling loop.
env = EnvironmentProxyForClient(receiver=None)
np.random.seed(1974)
env.seed(1974)
env.reset()
# nb_actions = cpst._action_space
nb_actions = 2
# Next, we build a very simple model.
model = Sequential()
#n_os = cpst._observation_space.shape
n_os = 4
model.add(Flatten(input_shape=[1] +[n_os]))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(nb_actions))
model.add(Activation('linear'))
print(model.summary())
# SARSA does not require a memory.
policy = BoltzmannQPolicy()
sarsa = SARSAAgent(model=model, nb_actions=nb_actions, nb_steps_warmup=10, policy=policy)
sarsa.compile(Adam(lr=1e-3), metrics=['mae'])
run_test(sarsa, env, log=log)
def keras_rl(env,
model_name,
saved_model_name="model",
steps=50000,
test_steps=5,
visualize=False,
hidden_layers=3,
critic_hidden_layers=3):
nb_actions = 0
if (model_name == "DQN" or model_name == "SARSA"):
nb_actions = env.action_space.n
elif (model_name == "DDPG"):
nb_actions = env.action_space.shape[0]
model_structure = define_layers(env,
nb_actions,
num_of_hidden_layers=hidden_layers)
memory = define_memory()
policy = define_policy(model_name)
if (model_name == "DQN"):
model = DQNAgent(model=model_structure,
nb_actions=nb_actions,
memory=memory,
nb_steps_warmup=100,
enable_double_dqn=True,
dueling_type='avg',
target_model_update=1e-2)
elif (model_name == "SARSA"):
model = SARSAAgent(model=model_structure,
nb_actions=nb_actions,
nb_steps_warmup=10,
policy=policy)
elif (model_name == "DDPG"):
action_input, critic_layers = define_critic_layers(
env, num_of_hidden_layers=critic_hidden_layers)
random_process = define_random_process(nb_actions)
model = DDPGAgent(nb_actions=nb_actions,
actor=model_structure,
critic=critic_layers,
critic_action_input=action_input,
memory=memory,
nb_steps_warmup_critic=100,
nb_steps_warmup_actor=100,
random_process=random_process,
gamma=.99,
target_model_update=1e-3)
model.compile(Adam(lr=1e-3), metrics=['mae'])
model.fit(env, nb_steps=steps, visualize=False, verbose=2)
model.save_weights('{}.h5f'.format(model_name), overwrite=True)
model.test(env, nb_episodes=test_steps, visualize=visualize)
def run_sarsa_agent(driver, queries, candidate_indices, tuning_config):
# Get the environment and extract the number of actions.
env = gym.make("udo_optimization-v0", driver=driver, queries=queries, candidate_indices=candidate_indices,
config=tuning_config)
env.horizon = tuning_config['horizon']
nb_actions = env.action_space.n
logging.info(f"nr action: {nb_actions}")
logging.info(f"observation space: {env.observation_space.shape}")
# Next, we build a very simple model.
model = Sequential()
model.add(Flatten(input_shape=(1,) + env.observation_space.shape))
model.add(Dense(52))
model.add(Activation('relu'))
model.add(Dense(252))
model.add(Activation('relu'))
model.add(Dense(526))
model.add(Activation('relu'))
model.add(Dense(252))
model.add(Activation('relu'))
model.add(Dense(nb_actions))
model.add(Activation('linear'))
logging.info(model.summary())
# SARSA does not require a memory.
policy = BoltzmannQPolicy()
# policy.select_action()
sarsa = SARSAAgent(model=model, nb_actions=nb_actions, nb_steps_warmup=10, policy=policy)
sarsa.compile(Adam(lr=1e-3), metrics=['mae'])
# Okay, now it's time to learn something! We visualize the training here for show, but this
# slows down training quite a lot. You can always safely abort the training prematurely using
# Ctrl + C.
sarsa.fit(env, nb_steps=500, visualize=False, verbose=2)
# After training is done, we save the final weights.
# sarsa.save_weights('sarsa_{}_weights.h5f'.format(udo_optimization-v0), overwrite=True)
# Finally, evaluate our algorithm for 5 episodes.
sarsa.test(env, nb_episodes=5, visualize=False)
env.print_state_summary(env.best_state)
def get_agent(agent_type, model_type, lr):
if agent_type == "sarsa":
policy = BoltzmannQPolicy()
model = get_model(model_type)
agent = SARSAAgent(model=model,
policy=policy,
nb_actions=nb_actions,
nb_steps_warmup=10,
gamma=0.99)
agent.compile(Adam(lr), metrics=['mae'])
return agent
elif agent_type == "dqn":
policy = BoltzmannQPolicy()
model = get_model(model_type)
memory = SequentialMemory(limit=50000, window_length=1)
agent = DQNAgent(model=model,
policy=policy,
nb_actions=nb_actions,
memory=memory,
nb_steps_warmup=10,
target_model_update=1e-2,
enable_double_dqn=True)
agent.compile(Adam(lr), metrics=['mae'])
return agent
elif agent_type == "a2c":
agent = A2CAgent(nb_actions,
len(env.observation_space.high),
nb_steps_warmup=10,
actor_lr=0.001,
critic_lr=0.005)
agent.compile(Adam(lr))
return agent
elif agent_type == "ppo":
pass
else:
print("Unsupported model")
exit(1)
def test_sarsa():
env = TwoRoundDeterministicRewardEnv()
np.random.seed(123)
env.seed(123)
random.seed(123)
nb_actions = env.action_space.n
# Next, we build a very simple model.
model = Sequential()
model.add(Dense(16, input_shape=(1,)))
model.add(Activation('relu'))
model.add(Dense(nb_actions, activation='linear'))
policy = EpsGreedyQPolicy(eps=.1)
sarsa = SARSAAgent(model=model, nb_actions=nb_actions, nb_steps_warmup=50, policy=policy)
sarsa.compile(Adam(lr=1e-3))
sarsa.fit(env, nb_steps=20000, visualize=False, verbose=0)
policy.eps = 0.
h = sarsa.test(env, nb_episodes=20, visualize=False)
assert_allclose(np.mean(h.history['episode_reward']), 3.)
def test_sarsa():
env = TwoRoundDeterministicRewardEnv()
np.random.seed(123)
env.seed(123)
random.seed(123)
nb_actions = env.action_space.n
# Next, we build a very simple model.
model = Sequential()
model.add(Dense(16, input_shape=(1,)))
model.add(Activation('relu'))
model.add(Dense(nb_actions, activation='linear'))
policy = EpsGreedyQPolicy(eps=.1)
sarsa = SARSAAgent(model=model, nb_actions=nb_actions, nb_steps_warmup=50, policy=policy)
sarsa.compile(Adam(lr=1e-3))
sarsa.fit(env, nb_steps=20000, visualize=False, verbose=0)
policy.eps = 0.
h = sarsa.test(env, nb_episodes=20, visualize=False)
assert_allclose(np.mean(h.history['episode_reward']), 3.)
nb_actions = env.action_space.n
# Next, we build a very simple model.
model = Sequential()
model.add(Flatten(input_shape=(1,) + env.observation_space.shape))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(nb_actions))
model.add(Activation('linear'))
print(model.summary())
# SARSA does not require a memory.
policy = BoltzmannQPolicy()
sarsa = SARSAAgent(model=model, nb_actions=nb_actions, nb_steps_warmup=10, policy=policy)
sarsa.compile(Adam(lr=1e-3), metrics=['mae'])
# Okay, now it's time to learn something! We visualize the training here for show, but this
# slows down training quite a lot. You can always safely abort the training prematurely using
# Ctrl + C.
sarsa.fit(env, nb_steps=50000, visualize=False, verbose=2)
# After training is done, we save the final weights.
sarsa.save_weights('sarsa_{}_weights.h5f'.format(ENV_NAME), overwrite=True)
# Finally, evaluate our algorithm for 5 episodes.
sarsa.test(env, nb_episodes=5, visualize=True)
env = gym.make('CartPole-v1')
states = env.observation_space.shape[0]
actions = env.action_space.n
def agent(states, actions):
model = Sequential()
model.add(Flatten(input_shape=(1, states)))
model.add(Dense(24, activation='relu'))
model.add(Dense(24, activation='relu'))
model.add(Dense(actions, activation='linear'))
return model
model = agent(env.observation_space.shape[0], env.action_space.n)
from rl.agents import SARSAAgent
from rl.policy import EpsGreedyQPolicy
sarsa = SARSAAgent(model=model,
policy=EpsGreedyQPolicy(),
nb_actions=env.action_space.n)
# sarsa.compile('adam', metrics = ['mse'])
# sarsa.fit(env, nb_steps = 50000, visualize = False, verbose = 1)
# scores = sarsa.test(env, nb_episodes = 100, visualize= True)
# sarsa.save_weights('1-sarsa_weights.h5f', overwrite=True)
sarsa.load_weights('1-sarsa_weights.h5f')
_ = sarsa.test(env, nb_episodes=100, visualize=True)
print('Average score over 100 test games:{}'.format(
np.mean(_.history['episode_reward'])))
class DistopiaSARSA:
def __init__(self,
env_name='distopia-initial4-v0',
in_path=None,
out_path=None,
terminate_on_fail=False,
reconstruct=False):
self.ENV_NAME = env_name
self.filename = self.ENV_NAME
self.init_paths(in_path, out_path)
self.init_env(terminate_on_fail)
self.init_model(reconstruct)
self.compile_agent()
def init_paths(self, in_path, out_path):
self.in_path = in_path #if self.in_path != None else './'
self.out_path = out_path if out_path != None else './'
self.log_path = "./logs/{}".format(time.time())
os.mkdir(self.log_path)
def init_env(self, terminate_on_fail):
self.env = gym.make(self.ENV_NAME)
self.env.terminate_on_fail = terminate_on_fail
self.env.record_path = "{}/ep_".format(self.log_path)
self.env = gym.wrappers.Monitor(self.env, "recording", force=True)
np.random.seed(234)
self.env.seed(234)
self.nb_actions = np.sum(self.env.action_space.nvec)
self.num_actions = self.env.NUM_DIRECTIONS
self.num_blocks = self.env.NUM_DISTRICTS * self.env.BLOCKS_PER_DISTRICT
def init_model(self, reconstruct=False):
if self.in_path != None:
if reconstruct == True:
self.construct_model()
else:
yaml_file = open(
"{}/{}.yaml".format(self.in_path, self.filename), 'r')
model_yaml = yaml_file.read()
yaml_file.close()
self.model = model_from_yaml(model_yaml)
self.model.load_weights("{}/{}.h5".format(self.in_path,
self.filename))
else:
# Next, we build a very simple model.
self.construct_model()
self.save_model()
print(self.model.summary())
def construct_model(self):
self.model = Sequential()
self.model.add(
Flatten(input_shape=(1, ) + self.env.observation_space.shape))
self.model.add(Dense(64))
self.model.add(Activation('relu'))
self.model.add(Dense(64))
self.model.add(Activation('relu'))
# self.model.add(Dense(16))
# self.model.add(Activation('relu'))
self.model.add(Dense(self.nb_actions))
self.model.add(Activation('linear'))
def save_model(self):
if self.out_path != None:
with open(self.filename + ".yaml", 'w+') as yaml_file:
yaml_file.write(self.model.to_yaml())
self.model.save_weights('{}/{}.h5'.format(self.out_path,
self.ENV_NAME))
def compile_agent(self):
# Finally, we configure and compile our agent. You can use every built-in Keras optimizer and
# even the metrics!
processor = DistopiaProcessor(self.num_blocks, self.num_actions)
#memory = SequentialMemory(limit=50000, window_length=1)
#policy = PatchedBoltzmannQPolicy(num_actions = self.num_actions, num_blocks = self.num_blocks)
#test_policy = PatchedGreedyQPolicy(num_actions = self.num_actions, num_blocks = self.num_blocks)
policy = BoltzmannQPolicy()
test_policy = GreedyQPolicy()
self.sarsa = SARSAAgent(model=self.model,
processor=processor,
nb_actions=self.nb_actions,
nb_steps_warmup=1000,
policy=policy,
test_policy=test_policy,
gamma=0.9)
self.sarsa.compile(Adam(lr=1e-3), metrics=['mae'])
def train(self, max_steps=100, episodes=100):
# Okay, now it's time to learn something! We visualize the training here for show, but this
# slows down training quite a lot. You can always safely abort the training prematurely using
# Ctrl + C.
self.env._max_steps = max_steps
#for i in range(episodes):
self.env.current_step = 0
n_steps = max_steps * episodes
logger = FileLogger(
filepath='{}/{}.json'.format(self.out_path, self.ENV_NAME))
self.sarsa.fit(self.env,
nb_steps=n_steps,
nb_max_episode_steps=max_steps,
visualize=False,
verbose=1,
callbacks=[logger])
#self.env.reset()
# After episode is done, we save the final weights.
self.sarsa.save_weights('{}/{}.h5'.format(self.out_path,
self.ENV_NAME),
overwrite=True)
def test(self):
# Finally, evaluate our algorithm for 5 episodes.
self.sarsa.test(self.env,
nb_episodes=5,
nb_max_start_steps=0,
visualize=True)
def agent(states, actions):
model = Sequential()
model.add(Flatten(input_shape=(1, states)))
model.add(Dense(24, activation="relu"))
model.add(Dense(24, activation="relu"))
model.add(Dense(24, activation="relu"))
model.add(Dense(actions, activation="linear"))
return model
model = agent(env.observation_space.shape[0], env.action_space.n)
policy = EpsGreedyQPolicy()
sarsa = SARSAAgent(model=model, policy=policy, nb_actions=env.action_space.n)
sarsa.compile("adam", metrics=["mse"])
sarsa.fit(env, nb_steps=10000, visualize=False, verbose=1)
ready()
scores = sarsa.test(env, nb_episodes=5, visualize=True)
print('Average score over 100 test games:{}'.format(np.mean(scores.history['episode_reward'])))
#sarsa.save_weights('sarsa_weights.h5f', overwrite=True) # save trained weights
model = Sequential()
model.add(Flatten(input_shape=(1, ) + env.observation_space.shape))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(nb_actions))
model.add(Activation('linear'))
print(model.summary())
# SARSA does not require a memory.
policy = BoltzmannQPolicy()
sarsa = SARSAAgent(model=model,
nb_actions=nb_actions,
nb_steps_warmup=10,
policy=policy)
sarsa.compile(Adam(lr=1e-3), metrics=['mae'])
# Okay, now it's time to learn something! We visualize the training here for show, but this
# slows down training quite a lot. You can always safely abort the training prematurely using
# Ctrl + C.
sarsa.fit(env, nb_steps=50000, visualize=False, verbose=2)
# After training is done, we save the final weights.
sarsa.save_weights('sarsa_{}_weights.h5f'.format(ENV_NAME), overwrite=True)
# Finally, evaluate our algorithm for 5 episodes.
sarsa.test(env, nb_episodes=10, visualize=True)
# Next, we build a very simple model.
model = Sequential()
model.add(Flatten(input_shape=(1, ) + env.observation_space.shape))
model.add(Dense(nb_actions))
model.add(Activation('linear'))
print(model.summary())
# Build the policy.
policy = BoltzmannQPolicy()
#policy = LinearAnnealedPolicy(EpsGreedyQPolicy(), attr='eps', value_max=1., value_min=.1, value_test=.05,
# nb_steps=10000)
if args.use_sarsa:
# SARSA does not require a memory.
agent = SARSAAgent(model=model,
nb_actions=nb_actions,
nb_steps_warmup=10,
policy=policy)
else:
memory = SequentialMemory(limit=50000, window_length=1)
agent = DQNAgent(model=model,
memory=memory,
nb_actions=nb_actions,
nb_steps_warmup=50,
policy=policy)
agent.compile(Adam(lr=args.learning_rate), metrics=['mae'])
# Okay, now it's time to learn something! We visualize the training here for show, but this
# slows down training quite a lot. You can always safely abort the training prematurely using
# Ctrl + C.
agent.fit(env, nb_steps=args.n_steps, visualize=False, verbose=2)
def train():
# Get the environment and extract the number of actions.
env = gym.make(ENV_NAME)
np.random.seed(123)
env.seed(123)
nb_actions = env.action_space.n
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
from keras import backend as K
K.set_session(sess)
# Next, we build a very simple model.
model = Sequential()
model.add(Flatten(input_shape=(1,) + env.observation_space.shape))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(nb_actions))
model.add(Activation('linear'))
print(model.summary())
# SARSA does not require a memory.
policy = BoltzmannQPolicy()
# processor_noisy = CartpoleSurrogateProcessor(e_= ERR_N, e=ERR_P, surrogate=False)
# processor_surrogate = CartpoleSurrogateProcessor(e_= ERR_N, e=ERR_P, surrogate=True)
if not SMOOTH:
processor_noisy = CartpoleProcessor(e_= ERR_N, e=ERR_P, smooth=False, surrogate=False)
processor_surrogate = CartpoleProcessor(e_= ERR_N, e=ERR_P, smooth=False, surrogate=True)
else:
processor_noisy = CartpoleProcessor(e_= ERR_N, e=ERR_P, smooth=True, surrogate=False)
processor_surrogate = CartpoleProcessor(e_= ERR_N, e=ERR_P, smooth=True, surrogate=True)
if REWARD == "normal":
sarsa_normal = SARSAAgent(model=model, nb_actions=nb_actions, nb_steps_warmup=10,
policy=policy)
sarsa_normal.compile(Adam(lr=1e-3), metrics=['mae'])
history_normal = sarsa_normal.fit(env, nb_steps=50000, visualize=False, verbose=2)
sarsa_normal.save_weights(os.path.join(LOG_DIR, 'sarsa_normal_{}_weights.h5f'.format(ENV_NAME)), overwrite=True)
sarsa_normal.test(env, nb_episodes=10, visualize=False, verbose=2)
pandas.DataFrame(history_normal.history).to_csv(os.path.join(LOG_DIR, "normal.csv"))
elif REWARD == "noisy":
sarsa_noisy = SARSAAgent(model=model, nb_actions=nb_actions, nb_steps_warmup=10,
policy=policy, processor=processor_noisy)
sarsa_noisy.compile(Adam(lr=1e-3), metrics=['mae'])
history_noisy = sarsa_noisy.fit(env, nb_steps=50000, visualize=False, verbose=2)
if not SMOOTH:
sarsa_noisy.save_weights(os.path.join(LOG_DIR, 'sarsa_noisy_{}_weights.h5f'.format(ENV_NAME)), overwrite=True)
pandas.DataFrame(history_noisy.history).to_csv(os.path.join(LOG_DIR, "noisy.csv"))
else:
sarsa_noisy.save_weights(os.path.join(LOG_DIR, 'sarsa_noisy_smooth_{}_weights.h5f'.format(ENV_NAME)), overwrite=True)
pandas.DataFrame(history_noisy.history).to_csv(os.path.join(LOG_DIR, "noisy_smooth.csv"))
sarsa_noisy.test(env, nb_episodes=10, visualize=False)
elif REWARD == "surrogate":
sarsa_surrogate = SARSAAgent(model=model, nb_actions=nb_actions, nb_steps_warmup=10,
policy=policy, processor=processor_surrogate)
sarsa_surrogate.compile(Adam(lr=1e-3), metrics=['mae'])
history_surrogate = sarsa_surrogate.fit(env, nb_steps=50000, visualize=False, verbose=2)
if not SMOOTH:
sarsa_surrogate.save_weights(os.path.join(LOG_DIR, 'sarsa_surrogate_{}_weights.h5f'.format(ENV_NAME)), overwrite=True)
pandas.DataFrame(history_surrogate.history).to_csv(os.path.join(LOG_DIR, "surrogate.csv"))
else:
sarsa_surrogate.save_weights(os.path.join(LOG_DIR, 'sarsa_surrogate_smooth_{}_weights.h5f'.format(ENV_NAME)), overwrite=True)
pandas.DataFrame(history_surrogate.history).to_csv(os.path.join(LOG_DIR, "surrogate_smooth.csv"))
sarsa_surrogate.test(env, nb_episodes=10, visualize=False)
def __init__(
self,
env="CartPole-v1",
emulateOculus=True,
visualize=True,
teachingFilesPath=None,
policyValues={
"inner_policy": EpsGreedyQPolicy(),
"attr": "eps",
"value_max": 0.75,
"value_min": .01,
"value_test": .0,
"nb_steps": 50000
},
dobotEmulation=False):
self.policyValues = policyValues
os.environ[
"PATH"] += os.pathsep + 'C:/Program Files (x86)/Graphviz2.38/bin/'
physical_devices = tf.config.experimental.list_physical_devices('GPU')
print("physical_devices-------------", len(physical_devices))
tf.config.experimental.set_memory_growth(physical_devices[0], True)
self.episodeLength = 25
if env == "CartPole-v1":
self.env = gym.make('CartPole-v1')
self.states = self.env.observation_space.shape[0]
self.actions = self.env.action_space.n
self.saveFileName = 'sarsa_weights.h5f'
logdir = "logs/CartPoleV1/" + datetime.now().strftime(
"%Y%m%d-%H%M%S")
self.tensorboard_callback = keras.callbacks.TensorBoard(
log_dir=logdir)
self.visualize = True
elif env == "Dobot":
self.env = dobotGym.dobotGym(emulateOculus=emulateOculus,
episodeLength=self.episodeLength,
visualize=visualize,
teachingFilesPath=teachingFilesPath,
dobotEmulation=dobotEmulation)
self.states = self.env.observation_space.shape[0]
self.actions = self.env.action_space.shape[0]
self.saveFileName = 'sarsa_weights_dobot.h5f'
logdir = "logs/Dobot/" + datetime.now().strftime("%Y%m%d-%H%M%S")
self.tensorboard_callback = keras.callbacks.TensorBoard(
log_dir=logdir)
self.visualize = True
else:
raise TypeError("Wrong env")
print(
'States', self.states
) # To get an idea about the number of variables affecting the environment
print(
'Actions', self.actions
) # To get an idea about the number of possible actions in the environment, do [right,left]
#
# episodes = 10
# for episode in range(1, episodes + 1):
# # At each begining reset the game
# state = self.env.reset()
# # set done to False
# done = False
# # set score to 0
# score = 0
# # while the game is not finished
# while not done:
# # visualize each step
# self.env.render()
# # choose a random action
# action = random.choice([0, 1])
# # execute the action
# n_state, reward, done, info = self.env.step(action)
# # keep track of rewards
# score += reward
# print('episode {} score {}'.format(episode, score))
# not working :(
# self.agent = self.agentDDP(self.states, self.actions)
# self.agent = self.NAFAgent(self.states, self.actions)
# self.policy = EpsGreedyQPolicy()
self.savingFreq = 100
self.actualSaving = 0
self.model = self.agentSarsa(self.states, self.actions)
self.policy = LinearAnnealedPolicy(
inner_policy=self.policyValues["inner_policy"],
attr=self.policyValues["attr"],
value_max=self.policyValues["value_max"],
value_min=self.policyValues["value_min"],
value_test=self.policyValues["value_test"],
nb_steps=self.policyValues["nb_steps"])
self.agent = SARSAAgent(model=self.model,
policy=self.policy,
nb_actions=self.actions)
self.agent._is_graph_network = True
def t():
return False
self.agent._in_multi_worker_mode = t
self.agent.save = self.saveAgentWeights
def lenmeh():
return self.actions
# Next, we build a very simple model.
model = Sequential()
model.add(Flatten(input_shape=(1,) + env.observation_space.shape))
model.add(Dense(8))
model.add(Activation('relu'))
model.add(Dense(8))
model.add(Activation('relu'))
model.add(Dense(8))
model.add(Activation('relu'))
model.add(Dense(nb_actions))
model.add(Activation('linear'))
print(model.summary())
# SARSA does not require a memory.
policy = BoltzmannQPolicy()
sarsa = SARSAAgent(model=model, nb_actions=nb_actions, nb_steps_warmup=10, policy=policy)
sarsa.compile(Adam(lr=1e-3), metrics=['mae'])
model_fn = 'sarsa_{}_weights.h5f'.format(ENV_NAME);
if os.path.isfile(model_fn):
sarsa.load_weights(model_fn)
# Okay, now it's time to learn something! We visualize the training here for show, but this
# slows down training quite a lot. You can always safely abort the training prematurely using
# Ctrl + C.
#sarsa.fit(env, nb_steps=50000,nb_max_episode_steps=500, visualize=False, verbose=2)
# After training is done, we save the final weights.
#sarsa.save_weights('sarsa_{}_weights.h5f'.format(ENV_NAME), overwrite=True)
# Finally, evaluate our algorithm for 5 episodes.
sarsa.test(env, nb_episodes=5, visualize=True)
def agent(states, actions):
model = Sequential()
model.add(Flatten(input_shape=(1, states)))
model.add(Dense(24, activation='relu'))
model.add(Dense(24, activation='relu'))
model.add(Dense(24, activation='relu'))
model.add(Dense(actions, activation='linear'))
return model
model = agent(env.observation_space.shape[0], env.action_space.n)
from rl.agents import SARSAAgent
from rl.policy import EpsGreedyQPolicy
policy = EpsGreedyQPolicy()
sarsa = SARSAAgent(model=model, policy=policy, nb_actions=env.action_space.n)
sarsa.compile('adam', metrics=['mse'])
sarsa.fit(env, nb_steps=50000, visualize=False, verbose=1)
scores = sarsa.test(env, nb_episodes=100, visualize=False)
print('Average score over 100 test games:{}'.format(
np.mean(scores.history['episode_reward'])))
sarsa.save_weights('sarsa_weights.h5f', overwrite=True)
_ = sarsa.test(env, nb_episodes=2, visualize=True)
env.close()
from rl.agents import SARSAAgent from rl.policy import EpsGreedyQPolicy() ENV_NAME = 'CartPole-v0' env = gym.make(ENV_NAME) np.random.seed(123) env.seed(123) nb_actions = env.action_space.n model = Sequential() model.add(Flatten(input_shape=(1,) + env.observation_space.shape)) model.add(Dense(24,activation='relu')) model.add(Dense(24, activation='relu')) model.add(Dense(nb_actions, activation='linear')) policy = EpsGreedyQPolicy() sarsa = SARSAAgent(model=model, nb_actions=nb_actions, nb_steps_warmup=10, policy=policy) sarsa.compile(Adam(lr=1e-2), metrics=['mae']) sarsa.fit(env, nb_steps=20000, visualize=False, verbose=2) sarsa.test(env, nb_episodes=5, visualize=True)
# agent network
model = Sequential()
model.add(Flatten(input_shape=(1, ) + env.observation_space.shape))
model.add(Dense(32 * scale))
model.add(Activation('relu'))
model.add(Dense(16 * scale))
model.add(Activation('relu'))
model.add(Dense(8 * scale))
model.add(Activation('relu'))
model.add(Dense(nb_actions, activation='softmax'))
print(model.summary())
# spcifications for the RL agent
policy = EpsGreedyQPolicy()
sarsa = SARSAAgent(model=model,
nb_actions=nb_actions,
nb_steps_warmup=1000,
policy=policy)
sarsa.compile(Adam(lr=1e-3), metrics=['mae'])
# compiling the model
sarsa.compile(Adam(lr=lrn_rate), metrics=['mae'])
# setting up callbacks for result collection and realtime visualization of the results through tensorboard
tensorboard = TensorBoard(log_dir="logs/{}".format(time()))
tpl = TrainEpisodeLogger()
# finally perform the training----- visualize=False enables training without visualizing the game which speeds up the training process
sarsa.fit(env,
nb_steps=nb_steps,
visualize=False,
verbose=2,
# Make a neural net with 3 hidden layers
def agent(states, actions):
model = Sequential()
model.add(Flatten(input_shape=(1, states)))
model.add(Dense(24, activation='relu'))
model.add(Dense(24, activation='relu'))
model.add(Dense(24, activation='relu'))
model.add(Dense(actions, activation='linear'))
return model
# Actually make a neural net with 3 hidden layers
model = agent(env.observation_space.shape[0], env.action_space.n)
policy = EpsGreedyQPolicy()
# Create a tensorflow reinforcement learning agent using the [state > action > reward] system
sarsa = SARSAAgent(model=model, policy=policy, nb_actions=env.action_space.n)
# Choose how we calculate reward and modify the model
sarsa.compile('adam', metrics=['mse'])
# sarsa.fit(env, nb_steps = 50000, visualize = False, verbose = 1)
sarsa.load_weights('cartpolekerassarsa.h5f')
scores = sarsa.test(env, nb_episodes=10, visualize=False)
print('Average score over 10 test games: {}'.format(
np.mean(scores.history['episode_reward'])))
sarsa.save_weights('cartpolekerassarsa.h5f', overwrite=True)
sarsa.test(env, nb_episodes=2, visualize=True)
class DQN:
def __init__(
self,
env="CartPole-v1",
emulateOculus=True,
visualize=True,
teachingFilesPath=None,
policyValues={
"inner_policy": EpsGreedyQPolicy(),
"attr": "eps",
"value_max": 0.75,
"value_min": .01,
"value_test": .0,
"nb_steps": 50000
},
dobotEmulation=False):
self.policyValues = policyValues
os.environ[
"PATH"] += os.pathsep + 'C:/Program Files (x86)/Graphviz2.38/bin/'
physical_devices = tf.config.experimental.list_physical_devices('GPU')
print("physical_devices-------------", len(physical_devices))
tf.config.experimental.set_memory_growth(physical_devices[0], True)
self.episodeLength = 25
if env == "CartPole-v1":
self.env = gym.make('CartPole-v1')
self.states = self.env.observation_space.shape[0]
self.actions = self.env.action_space.n
self.saveFileName = 'sarsa_weights.h5f'
logdir = "logs/CartPoleV1/" + datetime.now().strftime(
"%Y%m%d-%H%M%S")
self.tensorboard_callback = keras.callbacks.TensorBoard(
log_dir=logdir)
self.visualize = True
elif env == "Dobot":
self.env = dobotGym.dobotGym(emulateOculus=emulateOculus,
episodeLength=self.episodeLength,
visualize=visualize,
teachingFilesPath=teachingFilesPath,
dobotEmulation=dobotEmulation)
self.states = self.env.observation_space.shape[0]
self.actions = self.env.action_space.shape[0]
self.saveFileName = 'sarsa_weights_dobot.h5f'
logdir = "logs/Dobot/" + datetime.now().strftime("%Y%m%d-%H%M%S")
self.tensorboard_callback = keras.callbacks.TensorBoard(
log_dir=logdir)
self.visualize = True
else:
raise TypeError("Wrong env")
print(
'States', self.states
) # To get an idea about the number of variables affecting the environment
print(
'Actions', self.actions
) # To get an idea about the number of possible actions in the environment, do [right,left]
#
# episodes = 10
# for episode in range(1, episodes + 1):
# # At each begining reset the game
# state = self.env.reset()
# # set done to False
# done = False
# # set score to 0
# score = 0
# # while the game is not finished
# while not done:
# # visualize each step
# self.env.render()
# # choose a random action
# action = random.choice([0, 1])
# # execute the action
# n_state, reward, done, info = self.env.step(action)
# # keep track of rewards
# score += reward
# print('episode {} score {}'.format(episode, score))
# not working :(
# self.agent = self.agentDDP(self.states, self.actions)
# self.agent = self.NAFAgent(self.states, self.actions)
# self.policy = EpsGreedyQPolicy()
self.savingFreq = 100
self.actualSaving = 0
self.model = self.agentSarsa(self.states, self.actions)
self.policy = LinearAnnealedPolicy(
inner_policy=self.policyValues["inner_policy"],
attr=self.policyValues["attr"],
value_max=self.policyValues["value_max"],
value_min=self.policyValues["value_min"],
value_test=self.policyValues["value_test"],
nb_steps=self.policyValues["nb_steps"])
self.agent = SARSAAgent(model=self.model,
policy=self.policy,
nb_actions=self.actions)
self.agent._is_graph_network = True
def t():
return False
self.agent._in_multi_worker_mode = t
self.agent.save = self.saveAgentWeights
def lenmeh():
return self.actions
# self.agent.__len__ = lenmeh
def saveAgentWeights(self, path, overwrite=True):
if self.actualSaving < self.savingFreq:
self.actualSaving += 1
return None
else:
self.actualSaving = 0
path = 'model/checkpoint/' + datetime.now().strftime(
"%Y%m%d-%H%M%S") + self.saveFileName
self.agent.save_weights(path, overwrite)
def agentSarsa(self, states, actions):
self.model = Sequential()
self.model.add(LSTM(42, activation='sigmoid', input_shape=(1, states)))
self.model.add(Dense(42, activation='sigmoid'))
self.model.add(Dense(42, activation='sigmoid'))
self.model.add(Dense(24, activation='sigmoid'))
self.model.add(Dense(12, activation='sigmoid'))
self.model.add(Dense(actions, activation='linear'))
self.path = fileOperation.saveToFolder(self.model.to_json(),
name='modelShape',
folder="model\\checkpoint")
# , stateful=False states are resetted together after each batch.
# model.add(Flatten(input_shape=(1, states)))
# dot_img_file = '/model_1.png'
# keras.utils.plot_model(self.model, to_file=dot_img_file, show_shapes=True)
# model.reset_states()
return self.model
def load(self):
path = fileOperation.openDialogFunction(".h5f")
self.agent.compile('adam', metrics=['mse'])
self.agent.load_weights(path)
self.agent.compile('adam', metrics=['mse'])
def test(self, nb_episodes=2):
_ = self.agent.test(self.env,
nb_episodes=nb_episodes,
visualize=self.visualize)
def fit(self, visualize=False):
checkpoint_filepath = 'model/checkpoint/'
model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_filepath,
save_weights_only=False,
save_freq=25)
self.agent.compile('adam', metrics=['mse'])
self.agent.fit(
self.env,
nb_steps=self.policyValues["nb_steps"],
log_interval=self.episodeLength,
visualize=visualize,
verbose=1,
nb_max_start_steps=1,
start_step_policy=self.model.reset_states,
# callbacks=[PlotLossesKeras()])
callbacks=[self.tensorboard_callback, model_checkpoint_callback],
)
scores = self.agent.test(self.env, nb_episodes=5, visualize=visualize)
print('Average score over 5 test games:{}'.format(
np.mean(scores.history['episode_reward'])))
if nHiddenLayers > 1:
for i in range(1, nHiddenLayers):
hiddenLayer = Dense(nHiddenLayerNodes,
activation='relu',
kernel_initializer=weight_initializer)(hiddenLayer)
outputLayer = Dense(nb_actions, activation='linear')(hiddenLayer)
model = Model(inputLayer, outputLayer)
print(model.summary())
# SARSA does not require a memory.
policy = BoltzmannQPolicy()
sarsa = SARSAAgent(model=model,
nb_actions=nb_actions,
nb_steps_warmup=10,
policy=policy)
sarsa.compile(Adam(lr=1e-3), metrics=['mae'])
if loadFromExisting:
sarsa.load_weights(file_path)
else:
startTime = time.time()
sarsa.fit(env, nb_steps=nSteps, visualize=True, verbose=1)
endTime = time.time()
sarsa.save_weights(file_path, overwrite=True)
# After training is done, we save the final weights.
# Finally, evaluate our algorithm for 5 episodes.
sarsa.test(env, nb_episodes=5, visualize=True)
model.add(Flatten(input_shape=(1, states)))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(actions))
model.add(Activation('linear'))
return model
model = agent(states, actions)
# Define the policy
policy = EpsGreedyQPolicy()
# Define SARSA agent by feeding it the policy and the model
sarsa = SARSAAgent(model=model,
nb_actions=actions,
nb_steps_warmup=10,
policy=policy)
# compile sarsa with mean squared error loss
sarsa.compile('adam', metrics=['mse'])
# train the agent for 50000 steps
sarsa.fit(env, nb_steps=50000, visualize=False, verbose=1)
# Evaluate the agent on 100 new episodes.
scores = sarsa.test(env, nb_episodes=100, visualize=False)
print('Average score over 100 test games: {}'.format(
np.mean(scores.history['episode_reward'])))
