def __init__(self,
state_size,
action_size,
seed,
hidden_layers=[64, 64],
buffer_size=int(1e5),
batch_size=64,
gamma=0.99,
tau=1e-3,
learning_rate=5e-4,
update_every=4):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
seed (int): random seed
hidden_layers (list of int ; optional): number of each layer nodes
buffer_size (int ; optional): replay buffer size
batch_size (int; optional): minibatch size
gamma (float; optional): discount factor
tau (float; optional): for soft update of target parameters
learning_rate (float; optional): learning rate
update_every (int; optional): how often to update the network
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(seed)
self.buffer_size = buffer_size
self.batch_size = batch_size
self.gamma = gamma
self.tau = tau
self.lr = learning_rate
self.update_every = update_every
# detect GPU device
self.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
# Q-Network
model_params = [state_size, action_size, seed, hidden_layers]
self.qnetwork_local = QNetwork(*model_params).to(self.device)
self.qnetwork_target = QNetwork(*model_params).to(self.device)
self.optimizer = optim.Adam(self.qnetwork_local.parameters(),
lr=self.lr)
# Replay memory
self.memory = ReplayBuffer(action_size, self.buffer_size,
self.batch_size, seed, self.device)
# Initialize time step (for updating every self.update_every steps)
self.t_step = 0
def __init__(self,
state_size,
action_size,
seed,
buffer_size=int(1e5),
batch_size=64,
gamma=0.99,
tau=1e-3,
lr=5e-4,
update_every=4,
checkpoint_file='checkpoint.pth'):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
seed (int): random seed
buffer_size(int): replay buffer size
batch_size(int): minibatch size
gamma: discount factor
tau: for soft update of target parameters
lr: learning rate
update_every: how often to update the network
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(seed)
self.buffer_size = buffer_size
self.batch_size = batch_size
self.gamma = gamma
self.tau = tau
self.lr = lr
self.update_every = update_every
self.checkpoint_file = checkpoint_file
self.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
# Q-Network
self.qnetwork_local = QNetwork(state_size, action_size,
seed).to(self.device)
self.qnetwork_target = QNetwork(state_size, action_size,
seed).to(self.device)
self.optimizer = optim.Adam(self.qnetwork_local.parameters(), lr=lr)
# Replay memory
self.memory = ReplayBuffer(action_size, buffer_size, batch_size, seed,
self.device)
# Initialize time step (for updating every UPDATE_EVERY steps)
self.t_step = 0
def train(agent,
n_episodes=2000, max_t=1000, eps_start=1.0, eps_end=0.01, eps_decay=0.995):
"""Deep Q-Learning.
Params
======
agent: agent to train
n_episodes (int): maximum number of training episodes
max_t (int): maximum number of timesteps per episode
eps_start (float): starting value of epsilon, for epsilon-greedy action selection
eps_end (float): minimum value of epsilon
eps_decay (float): multiplicative factor (per episode) for decreasing epsilon
"""
scores = [] # list containing scores from each episode
scores_window = deque(maxlen=100) # last 100 scores
eps = eps_start # initialize epsilon
qnetwork_star = QNetwork(state_size=8, action_size=4, seed=0)
qnetwork_star.load_state_dict(torch.load('dqn_16.pth'))
qnetwork_star.eval()
Q_threshold = 0.0
savenumber = 0
for i_episode in range(1, n_episodes+1):
state = env.reset()
score = 0
for t in range(max_t):
action = agent.act(state, eps)
# shared autonomy
to_add = True
with torch.no_grad():
state = torch.from_numpy(state).float()
Q_values = qnetwork_star(state).data.numpy()
action_star = np.argmax(Q_values)
loss = Q_values[action_star] - Q_values[action]
if loss > Q_threshold:
to_add = False
action = action_star
next_state, reward, done, _ = env.step(action)
agent.step(state, action, reward, next_state, done, to_add)
state = next_state
score += reward
if done:
break
Q_threshold += 0.1
scores_window.append(score) # save most recent score
scores.append(score) # save most recent score
eps = max(eps_end, eps_decay*eps) # decrease epsilon
print('\rEpisode {}\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)), end="")
if i_episode % 100 == 0:
savename = "assisted_" + str(savenumber) + ".pkl"
print('\rEpisode {}\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)))
torch.save(agent.qnetwork_local.state_dict(), savename)
savenumber += 1
return scores
def main():
env = gym.make("LanderCustom-v0")
qnetwork = QNetwork(state_size=8, action_size=4, seed=0)
qnetwork.load_state_dict(torch.load('basic_lander.pth'))
qnetwork.eval()
human = MLP()
human.load_state_dict(torch.load('expert_bc.pt'))
human.eval()
softmax = torch.nn.Softmax(dim=0)
episodes = 30
scores = []
Q_threshold = 1e-2
for episode in range(episodes):
if episode < 10:
force_x = 0.0
elif episode < 20:
force_x = +500.0
else:
force_x = -500.0
env.start_state(force_x, 0.0)
state = env.reset()
score = 0
while True:
with torch.no_grad():
state = torch.from_numpy(state).float()
Q_values = qnetwork(state).data.numpy()
action_pred_dist = softmax(human(state).data).numpy()
action_star = np.argmax(Q_values)
action = np.random.choice(np.arange(4), p=action_pred_dist)
loss = Q_values[action_star] - Q_values[action]
# if loss > Q_threshold:
# action = action_star
# env.render()
state, reward, done, _ = env.step(action)
score += reward
if done:
print("episode: ", episode, "score: ", score)
break
scores.append(score)
env.close()
print("The average score is: ", np.mean(np.array(scores)))
def __init__(self, state_size, action_size, seed):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(seed)
# Q-Network
self.qnetwork_local = QNetwork(state_size, action_size, seed).to(device)
self.qnetwork_target = QNetwork(state_size, action_size, seed).to(device)
self.optimizer = optim.Adam(self.qnetwork_local.parameters(), lr=LR)
# Replay memory
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, seed)
# Initialize time step (for updating every UPDATE_EVERY steps)
self.t_step = 0
def rollout(force_x, Q_threshold, modelname):
env = gym.make("LanderCustom-v0")
qnetwork = QNetwork(state_size=8, action_size=4, seed=0)
qnetwork.load_state_dict(torch.load('basic_lander.pth'))
qnetwork.eval()
softmax = torch.nn.Softmax(dim=0)
if modelname is not None:
human = MLP()
human.load_state_dict(torch.load(modelname))
human.eval()
env.start_state(force_x, 0.0)
state = env.reset()
score = 0
dataset = []
while True:
# get robot and human actions
with torch.no_grad():
state = torch.from_numpy(state).float()
Q_values = qnetwork(state).data.numpy()
action_star = np.argmax(Q_values)
action = np.random.choice(np.arange(4))
if modelname is not None:
action_pred_dist = softmax(human(state).data).numpy()
action = np.random.choice(np.arange(4), p=action_pred_dist)
# save data
loss = Q_values[action_star] - Q_values[action]
dataset.append(list(state.numpy()) + [action, loss, action_star])
# shared autonomy
if loss > Q_threshold:
action = action_star
# update environment
# env.render()
state, reward, done, _ = env.step(action)
score += reward
if done:
# print("score: ", score)
break
env.close()
return score, dataset
def main():
env = gym.make("LanderCustom-v0")
fx_init = float(sys.argv[1])
Q_threshold = float(sys.argv[2])
savename = 'test1.pkl'
joystick = Joystick()
qnetwork = QNetwork(state_size=8, action_size=4, seed=0)
qnetwork.load_state_dict(torch.load('basic_lander.pth'))
qnetwork.eval()
human = MLP()
human.load_state_dict(torch.load('mlp_model.pt'))
human.eval()
episodes = 10
scores = []
data = []
env.start_state(fx_init, 0.0)
for episode in range(episodes):
state = env.reset()
env.render()
score = 0
while True:
action, start, stop = joystick.input()
if start:
break
while True:
action, start, stop = joystick.input()
data.append(list(state) + [action])
with torch.no_grad():
state = torch.from_numpy(state).float()
Q_values = qnetwork(state).data.numpy()
action_pred_dist = human(state).data.numpy()
action_star = np.argmax(Q_values)
action_pred = np.argmax(action_pred_dist)
# action = action_pred
loss = Q_values[action_star] - Q_values[action]
if loss > Q_threshold:
action = action_star
env.render()
state, reward, done, _ = env.step(action)
score += reward
if done or stop:
print(episode, score)
# pickle.dump(data, open(savename, "wb" ))
break
time.sleep(0.025)
scores.append(score)
env.close()
print(scores)
score_train, experiences = rollout(sa, mydata)
score_test, _ = rollout(1e3, mydata)
mytrainscore.append(score_train)
mytestscore.append(score_test)
learned_experiences = []
corrections = 0
for item in experiences:
state, loss, accepted, action, action_star = item[0:4], item[
4], item[5], item[6], item[7]
if not accepted:
corrections += 1
if loss > 0.5:
learned_experiences.append(state + [action_star])
mydata = mydata + learned_experiences
print(count, " TrainScore: ", score_train, " TestScore: ", \
score_test, " Correction: ", corrections, " Data: ", len(mydata))
return mytrainscore, mytestscore
if __name__ == "__main__":
env = gym.make("CartPole-v0")
env.seed(0)
qnetwork = QNetwork(state_size=4, action_size=2, seed=0)
qnetwork.load_state_dict(torch.load('models/dqn_cartpole.pth'))
qnetwork.eval()
scores = []
for idx in range(25):
mytrainscore, mytestscore = experience_matching()
scores.append(mytestscore)
pickle.dump(scores, open("results/failure/SomeSA.pkl", "wb"))
def dqn_main():
# Hyper parameters
lr = 1e-3
epochs = args.epoch
start_training = 1000
gamma = 0.99
batch_size = args.batch_size
epsilon = 1
epsilon_decay = .9999
target_update = 1000
learn_frequency = 2
env = Env(args)
# Init environment
state_size = env.state_size
action_size = env.action_space
# Init networks
q_network = QNetwork(state_size, action_size).cuda()
target_network = QNetwork(state_size, action_size).cuda()
target_network.load_state_dict(q_network.state_dict())
# Init optimizer
optim = torch.optim.Adam(q_network.parameters(), lr=lr)
# Init replay buffer
memory = []
# Begin main loop
results_dqn = []
global_step = 0
for epoch in range(epochs):
# Reset environment
state = env.reset()
done = False
cum_reward = 0 # Track cumulative reward per episode
frame = 0
# Begin episode
for step in tqdm(range(args.num_frames),
desc='Episode: {} Reward: {}'.format(
epoch, cum_reward)):
if done or cum_reward <= -200:
break
frame += 1
# Select e-greedy action
action, epsilon = get_action_dqn(q_network, state, epsilon,
epsilon_decay)
# Take step
next_state, reward, done, _ = env.step(action)
if done:
print('DONE!')
# env.render()
# Store step in replay buffer
memory.append(
(state.numpy(), action, next_state.numpy(), reward, done))
cum_reward += reward
global_step += 1 # Increment total steps
state = next_state # Set current state
# If time to train
if global_step > start_training and global_step % learn_frequency == 0:
# Sample batch
batch = prepare_batch(memory, batch_size)
# Train
learn_dqn(batch, optim, q_network, target_network, gamma,
global_step, target_update)
# Print results at end of episode
results_dqn.append(cum_reward)
return results_dqn
def play(n_episodes=100, max_t=1000):
# for first train eps start is 1
# for second eps start is 0.5
# also I changed max_t to 500, because it would take a long time to land
possible_actions = [
# don't move
[0, 0],
# up
[0.1, 0],
[0.2, 0],
[0.3, 0],
[0.5, 0],
[0.6, 0],
[0.7, 0],
[0.8, 0],
[0.9, 0],
[1, 0],
# left
[0, -0.6],
[0, -0.7],
[0, -0.8],
[0, -0.9],
# right
[0, 0.6],
[0, 0.7],
[0, 0.8],
[0, 0.9],
# up-left
# [0.8, -0.8],
# [0.8, -0.65],
# [0.6, -0.8],
# [0.6, -0.65],
# [0.7, -0.8],
# [0.7, -0.65],
# # up-right
# [0.8, 0.8],
# [0.8, 0.65],
# [0.6, 0.8],
# [0.6, 0.65],
# [0.7, 0.8],
# [0.7, 0.65]
]
env = gym.make('LunarLanderContinuous-v2')
# env = gym.make('LunarLander-v2')
# action_space = env.action_space
# print("action size=", action_space)
n_actions = len(possible_actions)
the_ship = QNetwork(state_size=8, action_size=n_actions)
if (loaded_model):
the_ship.load_state_dict(torch.load(loaded_model))
the_ship.eval()
scores_window = deque(maxlen=100)
state_arr = []
scores = []
# run n games
for i_episode in range(1, n_episodes + 1):
state = env.reset()
score = 0
# sample random transitions
for t in range(max_t):
state_arr = [state]
input_s = torch.tensor(state_arr)
qs = the_ship(input_s)
the_ship.train()
_, action = torch.max(qs.detach(), 1)
action = action.detach().numpy()[0]
env.render()
action_continious = discrete_action_to_continious_array(
possible_actions, action)
next_state, reward, done, _ = env.step(action_continious)
score += reward
state = next_state
if done:
break
scores_window.append(score)
scores.append(score)
mean_score = np.mean(scores_window)
print('\rEpisode {}\tAverage Score: {:.2f}, Last reward: {}'.format(
i_episode, mean_score, reward),
end="")
if i_episode % 100 == 0:
print(
'\rEpisode {}\tAverage Score: {:.2f}, Last reward: {}'.format(
i_episode, mean_score, reward))
return scores
score_test, _ = rollout(1e3, mydata)
mytrainscore.append(score_train)
mytestscore.append(score_test)
learned_experiences = []
corrections = 0
for item in experiences:
state, loss, accepted, action, action_star = item[0:6], item[6], item[7], item[8], item[9]
if not accepted:
corrections += 1
if accepted or not accepted:
learned_experiences.append(state + [action_star])
if len(learned_experiences) > 20:
learned_experiences = random.sample(learned_experiences, k=20)
mydata = mydata + learned_experiences
print(count, " TrainScore: ", score_train, " TestScore: ", \
score_test, " Correction: ", corrections, " Data: ", len(mydata))
return mytrainscore, mytestscore
if __name__ == "__main__":
env = gym.make("Acrobot-v1")
env.seed(0)
qnetwork = QNetwork(state_size=6, action_size=3, seed=0)
qnetwork.load_state_dict(torch.load('models/dqn_acrobot.pth'))
qnetwork.eval()
scores = []
for idx in range(25):
mytrainscore, mytestscore = experience_matching()
scores.append(mytestscore)
pickle.dump( scores, open( "results/obs/AllSA-Full.pkl", "wb" ) )