def main(model_path, n_test_episodes):
run_paths = glob.glob(os.path.join(model_path, '*'))
for run_path in run_paths:
if len(glob.glob(os.path.join(run_path, 'carracing_results*'))) > 0:
print(run_path, 'already processed')
continue
# Load run config
run_config = json.load(open(os.path.join(run_path, 'config.json'), 'r'))
env = gym.make("CarRacing-v0").unwrapped
num_actions = 5
# Define networks and load agent
if run_config['model'] == 'Resnet':
Q_net = ResnetVariant(num_actions=num_actions, history_length=run_config['history_length'] + 1).to(device)
Q_target_net = ResnetVariant(num_actions=num_actions, history_length=run_config['history_length'] + 1).to(
device)
elif run_config['model'] == 'Lenet':
Q_net = LeNetVariant(num_actions=num_actions, history_length=run_config['history_length'] + 1).to(device)
Q_target_net = LeNetVariant(num_actions=num_actions, history_length=run_config['history_length'] + 1).to(
device)
elif run_config['model'] == 'DeepQNetwork':
Q_net = DeepQNetwork(num_actions=num_actions, history_length=run_config['history_length'] + 1).to(device)
Q_target_net = DeepQNetwork(num_actions=num_actions, history_length=run_config['history_length'] + 1).to(
device)
else:
raise ValueError('{} not implmented.'.format(run_config['model']))
agent = DQNAgent(Q=Q_net, Q_target=Q_target_net, num_actions=num_actions, **run_config)
agent.load(os.path.join(run_path, 'agent.pt'))
episode_rewards = []
for i in range(n_test_episodes):
stats = run_episode(env, agent, deterministic=True, history_length=run_config['history_length'],
do_training=False, rendering=True, normalize_images=run_config['normalize_images'],
skip_frames=run_config['skip_frames'], max_timesteps=1000)
episode_rewards.append(stats.episode_reward)
# save results in a dictionary and write them into a .json file
results = dict()
results["episode_rewards"] = episode_rewards
results["mean"] = np.array(episode_rewards).mean()
results["std"] = np.array(episode_rewards).std()
fname = "{}/carracing_results_dqn-{}.json".format(run_path, datetime.now().strftime("%Y%m%d-%H%M%S"))
fh = open(fname, "w")
json.dump(results, fh)
fh.close()
env.close()
print('... finished')
def build(path):
# create world
world = World(path, thread_num=args.thread)
# create agents
agents = []
for i in world.intersections:
action_space = gym.spaces.Discrete(len(i.phases))
agents.append(
DQNAgent(
action_space,
LaneVehicleGenerator(world,
i, ["lane_count"],
in_only=True,
average=None),
LaneVehicleGenerator(world,
i, ["lane_waiting_count"],
in_only=True,
average="all",
negative=True), i.id))
if args.load_model:
agents[-1].load_model(args.save_dir)
# if len(agents) == 5:
# break
# create metric
metric = TravelTimeMetric(world)
# create env
env = TSCEnv(world, agents, metric)
return world, agents, env
def _single_dqn_test_demo():
catch_game_object = MultiPlayerCatch(1,
board_size=20,
food_spawn_rate=0.05)
visualizer = ImageStateVisualizator('MPCatch visualization', 10)
recorder = ImageStateRecorder('MPCatch_rgb_trained_network_results')
model = load_trained_model('final.h5')
agent = DQNAgent(model, 10000)
agent.train(catch_game_object,
epochs=100000,
batch_size=50,
gamma=0.9,
epsilon=0.1,
visualizer=visualizer)
def init(args, test=False):
tf_mute_warning()
args.save_dir = save_dir + args.config_file[7:-5]
if test:
args.save_dir = save_dir + args.config_file[7:-10]
# config_name = args.config_file.split('/')[1].split('.')[0]
# args.agent_save_dir = args.save_dir + "/" + config_name
if not os.path.exists(args.save_dir):
os.mkdir(args.save_dir)
if not os.path.exists(args.log_dir):
os.makedirs(args.log_dir)
logger = logging.getLogger('main')
logger.setLevel(logging.DEBUG)
fh = logging.FileHandler(
os.path.join(args.log_dir,
datetime.now().strftime('%Y%m%d-%H%M%S') + ".log"))
fh.setLevel(logging.DEBUG)
sh = logging.StreamHandler()
sh.setLevel(logging.INFO)
logger.addHandler(fh)
logger.addHandler(sh)
# create world
world = World(args.config_file, thread_num=args.thread, silent=True)
# create agents
agents = []
for i in world.intersections:
action_space = gym.spaces.Discrete(len(i.phases))
agents.append(
DQNAgent(
action_space,
LaneVehicleGenerator(world,
i, ["lane_count"],
in_only=True,
average=None),
LaneVehicleGenerator(world,
i, ["lane_waiting_count"],
in_only=True,
average="all",
negative=True), i.id))
if args.load_model:
agents[-1].load_model(args.save_dir)
if args.share_weights:
model = agents[0].model
for agent in agents:
agent.model = model
# create metric
metric = TravelTimeMetric(world)
# create env
env = TSCEnv(world, agents, metric)
return env
def create(self, model):
parameters = self._parameters
#行動の数
actions_count = 2
#方策(ここでいう方策とは、greedyかε-greedyということ)
policies = self._create_policies(model, parameters, actions_count)
#経験を記憶する
memory = Memory(parameters["memory_size"])
#割引率γ
gamma = parameters["gamma"]
#replay_start_memory_size個のデータが貯まるまで学習を開始しない
replay_start_memory_size = parameters["replay_start_memory_size"]
#experience_replayするときのデータ数
replay_count = parameters["replay_count"]
#学習する間隔を少し開ける
training_interval_steps = parameters["training_interval_steps"]
#optimizerを生成する
optimizer_parameters = (parameters["optimizer"]["alpha"],
parameters["optimizer"]["epsilon"])
optimizer = optimizers.Adam(alpha=optimizer_parameters[0],
eps=optimizer_parameters[1])
optimizer.setup(model)
#モデルを更新する処理
model_updater = SoftModelUpdater(parameters["tau"])
#agent生成
agent = DQNAgent(gamma, model, optimizer, model_updater, memory,
replay_start_memory_size, replay_count,
training_interval_steps, policies)
return agent
def make_random_agents():
return [DQNAgent(STATE_SIZE, N_ACTIONS, N_AGENTS, STARTING_EPSILON, E_MIN, E_DECAY, GAMMA)] * N_AGENTS
def copy_agent(agent):
weights = agent.model.get_weights()
copied_model = clone_model(agent.model)
copied = DQNAgent(*agent.get_init_info())
copied.set_model(copied_model, weights)
return copied
import numpy as np
np.random.seed(0)
if __name__ == "__main__":
env = gym.make("CartPole-v0").unwrapped
# TODO: load DQN agent
# ...
state_dim = 4
num_actions = 2
Q = MLP(state_dim, num_actions)
Q_target = MLP(state_dim, num_actions)
agent = DQNAgent(Q, Q_target, num_actions)
agent.load("C:\\Users\\Monish\\Desktop\\workspace\\exercise3_R\\reinforcement_learning\\models_cartpole\\dqn_agent_199.pt")
n_test_episodes = 15
episode_rewards = []
for i in range(n_test_episodes):
stats = run_episode(env, agent, deterministic=True, do_training=False, rendering=True)
episode_rewards.append(stats.episode_reward)
# save results in a dictionary and write them into a .json file
results = dict()
results["episode_rewards"] = episode_rewards
results["mean"] = np.array(episode_rewards).mean()
results["std"] = np.array(episode_rewards).std()
import numpy as np
np.random.seed(0)
if __name__ == "__main__":
env = gym.make("CartPole-v0").unwrapped
# TODO: load DQN agent
# ...
states_dim = 4
action_dim = 2
Q = MLP(states_dim, action_dim)
Q_target = MLP(states_dim, action_dim)
agent = DQNAgent(Q, Q_target, action_dim, double=True)
agent.load("./models_cartpole/dqn_agent_fixed_1.pt")
n_test_episodes = 15
episode_rewards = []
for i in range(n_test_episodes):
stats = run_episode(env,
agent,
deterministic=True,
do_training=False,
rendering=True)
episode_rewards.append(stats.episode_reward)
# save results in a dictionary and write them into a .json file
results = dict()
results["episode_rewards"] = episode_rewards
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("-m", "--model", type=str, help="Model file to use", required=True)
parser.add_argument('-e', "--episodes", type=int, help="num episodes to try", default=5, required=False)
args = parser.parse_args()
env = gym.make("CarRacing-v0").unwrapped
history_length = 5
#TODO: Define networks and load agent
# ....
Q_network = CNN(history_length=history_length, n_classes=5)
Q_target = CNN(history_length=history_length, n_classes=5)
agent = DQNAgent(Q=Q_network, Q_target=Q_target, num_actions=5)
agent.load(args.model)
episode_rewards = []
for i in range(args.episodes):
stats = run_episode(env, agent, deterministic=True, do_training=False, rendering=True, history_length=history_length)
episode_rewards.append(stats.episode_reward)
print('Episode %d - [ Reward %.2f ]' % (i+1, stats.episode_reward))
# save results in a dictionary and write them into a .json file
results = dict()
results["episode_rewards"] = episode_rewards
results["mean"] = np.array(episode_rewards).mean()
results["std"] = np.array(episode_rewards).std()
if not os.path.exists("./results"):
num_eval_episodes = 5 # evaluate on 5 episodes
eval_cycle = 10 # evaluate every 10 episodes
# You find information about cartpole in
# https://github.com/openai/gym/wiki/CartPole-v0
# Hint: CartPole is considered solved when the average reward is greater than or equal to 195.0 over 100 consecutive trials.
env = gym.make("CartPole-v0").unwrapped
state_dim = 4
num_actions = 2
# TODO:
# 1. init Q network and target network (see dqn/networks.py)
# 2. init DQNAgent (see dqn/dqn_agent.py)
# 3. train DQN agent with train_online(...)
Q = MLP(state_dim, num_actions)
Q_target = MLP(state_dim, num_actions)
agent = DQNAgent(Q,
Q_target,
num_actions,
gamma=0.9,
batch_size=32,
epsilon=0.1,
tau=0.01,
lr=0.001,
history_length=0)
training, validation = train_online(env, agent, 200, eval_cycle,
num_eval_episodes)
# store model.
if i % eval_cycle == 0 or i >= (num_episodes - 1):
agent.save(os.path.join(model_dir, "dqn_agent-3.pt"))
tensorboard.close_session()
if __name__ == "__main__":
num_eval_episodes = 5 # evaluate on 5 episodes
eval_cycle = 20 # evaluate every 10 episodes
# You find information about cartpole in
# https://github.com/openai/gym/wiki/CartPole-v0
# Hint: CartPole is considered solved when the average reward is greater than or equal to 195.0 over 100 consecutive trials.
env = gym.make("CartPole-v0").unwrapped
state_dim = 4
num_actions = 2
# TODO:
# 1. init Q network and target network (see dqn/networks.py)
Q = MLP(state_dim, num_actions)
Q_target = MLP(state_dim, num_actions)
# 2. init DQNAgent (see dqn/dqn_agent.py)
agent = DQNAgent(Q, Q_target, num_actions, history_length=1000000)
# 3. train DQN agent with train_online(...)
train_online(env=env, agent=agent, num_episodes=1000)
# if i % eval_cycle == 0 or i >= (num_episodes - 1):
tensorboard_train.close_session()
tensorboard_eval.close_session()
if __name__ == "__main__":
num_eval_episodes = 5 # evaluate on 5 episodes
eval_cycle = 10 # evaluate every 10 episodes
num_episodes = 10000
# You find information about cartpole in
# https://github.com/openai/gym/wiki/CartPole-v0
# Hint: CartPole is considered solved when the average reward is greater than or equal to 195.0 over 100 consecutive trials.
env = gym.make("CartPole-v0").unwrapped
state_dim = 4
num_actions = 2
# TODO:
# 1. init Q network and target network (see dqn/networks.py)
# ...
Q_target = MLP(state_dim, num_actions)
Q = MLP(state_dim, num_actions)
# 2. init DQNAgent (see dqn/dqn_agent.py)
# agent = DQNAgent(Q, Q_target, num_actions, double=True, history_length=1e6)
agent = DQNAgent(Q, Q_target, num_actions, double=True, epsilon=0.99, eps_decay=True, history_length=1e6)
# 3. train DQN agent with train_online(...)
train_online(env, agent, num_episodes, num_eval_episodes, eval_cycle)
if __name__ == "__main__":
# You find information about cartpole in
# https://github.com/openai/gym/wiki/CartPole-v0
# Hint: CartPole is considered solved when the average reward is greater than or equal to 195.0 over 100 consecutive trials.
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--interrupt", action='store_true', help="Save model if interrupted",
default=False, required=False)
parser.add_argument('-e', "--episodes", type=int, help="num episodes to try", default=500, required=False)
parser.add_argument('-s', "--steps", type=int, help="num steps per episode", default=200, required=False)
parser.add_argument("-r", "--render", action='store_true', help="render during training and evaluation",
default=False, required=False)
args = parser.parse_args()
print(args)
env = gym.make("CartPole-v0").unwrapped
state_dim = 4
num_actions = 2
# TODO:
# 1. init Q network and target network (see dqn/networks.py)
Q_network = MLP(state_dim, num_actions)
Q_target = MLP(state_dim, num_actions)
# 2. init DQNAgent (see dqn/dqn_agent.py)
agent = DQNAgent(Q=Q_network, Q_target=Q_target, num_actions=num_actions, buffer_size=1e5, lr=1e-4)
# 3. train DQN agent with train_online(...)
train_online(env=env, agent=agent, num_episodes=args.episodes, max_timesteps=args.steps,
eval_cycle=20, num_eval_episodes=5, rendering=args.render,
tensorboard_dir='./tensorboard', save_interrupt=args.interrupt)
def setup_ai(model_path):
agent = DQNAgent(STATE_SIZE, N_ACTIONS, N_AGENTS, EPSILON, None, None, 0.95)
agent.load(model_path)
return DQNAgentWrapper(agent, STACK_SIZE)
def setup_ai(model_path):
agent = DQNAgent(STATE_SIZE, N_ACTIONS, N_AGENTS, EPSILON, None, None,
0.95)
agent.load(model_path)
return DQNAgentWrapper(agent, STACK_SIZE)
# Hint: CartPole is considered solved when the average reward is greater than or equal to 195.0 over 100 consecutive trials.
env = gym.make("CartPole-v0").unwrapped
#import pdb; pdb.set_trace()
state_dim = 4
num_actions = 2
# TODO:
# 1. init Q network and target network (see dqn/networks.py)
# 2. init DQNAgent (see dqn/dqn_agent.py)
# 3. train DQN agent with train_online(...)
# Duelling DQN or Not
Duel = False
num_episodes = 2000
if Duel:
Q = MLP_Duel(state_dim, num_actions)
Q_target = MLP_Duel(state_dim, num_actions)
else:
Q = MLP(state_dim, num_actions)
Q_target = MLP(state_dim, num_actions)
DQNAgent = DQNAgent(Q,
Q_target,
num_actions,
double=True,
history_length=1e6)
train_online(env, DQNAgent, num_episodes, epsilon_decay=False)
np.random.seed(0)
if __name__ == "__main__":
env = gym.make("CartPole-v0").unwrapped
# TODO: load DQN agent
# ...
state_dim = 4
num_actions = 2
Q = MLP(state_dim, num_actions)
Q_target = MLP(state_dim, num_actions)
PATH = '/home/salem/Documents/freiburg/Lab/CarRacing/reinforcement_learning/models_cartpole/best_eval_dqn_agent.pt'
Q.load_state_dict(torch.load(PATH), strict=False)
Q_target.load_state_dict(torch.load(PATH), strict=False)
agent = DQNAgent(Q, Q_target, num_actions, double=True)
n_test_episodes = 150
episode_rewards = []
for i in range(n_test_episodes):
stats = run_episode(env,
agent,
eps=0.1,
deterministic=True,
do_training=False,
rendering=True)
print(stats.episode_reward)
episode_rewards.append(stats.episode_reward)
# save results in a dictionary and write them into a .json file
results = dict()
def __init__(self, *args, **kwargs):
DQNAgent.__init__(self, *args, **kwargs)
def setup_ai(model_path):
agent = DQNAgent(STATE_SIZE, N_ACTIONS, N_AGENTS, None, None, None)
agent.epsilon = 0.01
agent.load(model_path)
return DQNAgentWrapper(agent, STACK_SIZE)
from agent.dqn_agent import DQNAgent
from function import *
import sys
if len(sys.argv) != 4:
print("Usage: python train.py [stock] [window] [episodes]")
exit()
stock_name, window_size, episode_count = sys.argv[1], int(sys.argv[2]), int(sys.argv[3])
agent = DQNAgent(window_size)
data = getStockDataVec(stock_name)
len_data = len(data)
l = len(data) - 1
batch_size = 32
for e in range(episode_count + 1):
print("Episode " + str(e) + "/" + str(episode_count))
state = getState(data, 0, window_size + 1, len_data)
total_profits = []
total_profit = 0
agent.inventory = []
for t in range(l):
action = agent.act(state)
next_state = getState(data, t + 1, window_size + 1, len_data)
reward = 0
if action == 1: # buy
agent.inventory.append(data[t][1])
total_profits.append(['buy', total_profit])
logger.addHandler(sh)
# create world
world = World(args.config_file, thread_num=args.thread)
# create agents
agents = []
for i in world.intersections:
action_space = gym.spaces.Discrete(len(i.phases))
agents.append(
DQNAgent(
action_space,
LaneVehicleGenerator(world,
i, ["lane_count"],
in_only=True,
average=None),
LaneVehicleGenerator(world,
i, ["lane_waiting_count"],
in_only=True,
average="all",
negative=True), i.id))
if args.load_model:
agents[-1].load_model(args.save_dir)
# create metric
metric = TravelTimeMetric(world)
# create env
env = TSCEnv(world, agents, metric)
from train_cartpole import run_episode
from agent.networks import *
import numpy as np
from agent.networks import MLP
import torch
np.random.seed(0)
if __name__ == "__main__":
env = gym.make("CartPole-v0").unwrapped
# TODO: load DQN agent
# ...
Q = MLP(state_dim = 4,action_dim = 2)
Q_target = MLP(state_dim = 4, action_dim = 2)
agent = DQNAgent(Q, Q_target, num_actions = 2)
agent.load("./models_cartpole/dqn_agent-perfect.pt")
n_test_episodes = 15
episode_rewards = []
for i in range(n_test_episodes):
stats = run_episode(env, agent, deterministic=True, do_training=False, rendering=True,epsilon=0)
episode_rewards.append(stats.episode_reward)
# save results in a dictionary and write them into a .json file
results = dict()
results["episode_rewards"] = episode_rewards
results["mean"] = np.array(episode_rewards).mean()
results["std"] = np.array(episode_rewards).std()
if not os.path.exists("./results"):