def main():
# Parse arguments
parser = argparse.ArgumentParser()
parser.add_argument('-g', action='store', dest='game')
parser.add_argument('-w',
action='store_true',
dest='warm_start',
default=False)
args = parser.parse_args()
game = args.game
warm_start = args.warm_start
# Initialize environment
env = gym.make(game)
num_actions = env.action_space.n
# Initialize constants
num_frames = 4
max_episodes = 1000000
max_frames = 10000
gamma = 0.95
lr = 1e-4 # LSTM Update: Work well in 1st iteration
target_score = 21.0 # Temperature Update: specific to Pong
# Cold start
if not warm_start:
# Initialize model
model = Policy(input_channels=num_frames, num_actions=num_actions)
optimizer = optim.RMSprop(model.parameters(), lr=lr,
weight_decay=0.1) #LSTM Change: lr = 1e-4
# Initialize statistics
running_reward = None
running_rewards = []
prior_eps = 0
# Warm start
if warm_start:
data_file = 'results/{}.p'.format(game)
try:
with open(data_file, 'rb') as f:
running_rewards = pickle.load(f)
running_reward = running_rewards[-1]
prior_eps = len(running_rewards)
model_file = 'saved_models/actor_critic_{}_ep_{}.p'.format(
game, prior_eps)
with open(model_file, 'rb') as f:
# Model Save and Load Update: Include both model and optim parameters
saved_model = pickle.load(f)
model, optimizer = saved_model
except OSError:
print('Saved file not found. Creating new cold start model.')
model = Policy(input_channels=num_frames, num_actions=num_actions)
optimizer = optim.RMSprop(model.parameters(),
lr=lr,
weight_decay=0.1)
running_reward = None
running_rewards = []
prior_eps = 0
cuda = torch.cuda.is_available()
if cuda:
model = model.cuda()
for ep in range(max_episodes):
# Temperature Update: specific to Pong
# Anneal temperature from 2.0 down to 0.8 based on how far running reward is from
# target score
if running_reward is None:
model.temperature = 2.0 # Start with temp = 2.0 (Explore)
else:
# Specific to Pong - running reward starts at -21, so we encourage the agent
# to explore. temp = 0.8 + 1.2*[21-(-21)]/42 = 2.0
# As it gets closer to 0, temp = 0.8 + 1.2(21-0)/42 = 1.4
# As it gets to 7, temp = 0.8 + 1.2(21-14)/42 = 1.0
model.temperature = max(
0.8, 0.8 + (target_score - running_reward) / 42 * 1.2)
state = env.reset()
state = preprocess_state(state)
state = np.stack([state] * num_frames)
# LSTM change - reset LSTM hidden units when episode begins
cx = Variable(torch.zeros(1, 256))
hx = Variable(torch.zeros(1, 256))
if cuda:
cx = cx.cuda()
hx = hx.cuda()
reward_sum = 0.0
for frame in range(max_frames):
# Select action
# LSTM Change: Need to cycle hx and cx thru select_action
action, log_prob, state_value, (hx, cx) = select_action(
model, state, (hx, cx), cuda)
model.saved_actions.append((log_prob, state_value))
# Perform step
next_state, reward, done, info = env.step(action)
# Add reward to reward buffer
model.rewards.append(reward)
reward_sum += reward
# Compute latest state
next_state = preprocess_state(next_state)
# Evict oldest diff add new diff to state
next_state = np.stack([next_state] * num_frames)
next_state[1:, :, :] = state[:-1, :, :]
state = next_state
if done:
break
# Compute/display statistics
if running_reward is None:
running_reward = reward_sum
else:
running_reward = running_reward * 0.99 + reward_sum * 0.01
running_rewards.append(running_reward)
verbose_str = 'Episode {} complete'.format(ep + prior_eps + 1)
verbose_str += '\tReward total:{}'.format(reward_sum)
verbose_str += '\tRunning mean: {:.4}'.format(running_reward)
# Temperature Update: Track temp
if (ep + prior_eps + 1) % 5 == 0:
verbose_str += '\tTemp = {:.4}'.format(model.temperature)
sys.stdout.write('\r' + verbose_str)
sys.stdout.flush()
# Update model
finish_episode(model, optimizer, gamma, cuda)
if (ep + prior_eps + 1) % 500 == 0:
model_file = 'saved_models/actor_critic_{}_ep_{}.p'.format(
game, ep + prior_eps + 1)
data_file = 'results/{}.p'.format(game)
with open(model_file, 'wb') as f:
# Model Save and Load Update: Include both model and optim parameters
pickle.dump((model.cpu(), optimizer), f)
if cuda:
model = model.cuda()
with open(data_file, 'wb') as f:
pickle.dump(running_rewards, f)
def main():
# Parse arguments
parser = argparse.ArgumentParser()
parser.add_argument('-g', action='store', dest='game')
parser.add_argument('-w',
action='store_true',
dest='warm_start',
default=False)
args = parser.parse_args()
game = args.game
warm_start = args.warm_start
# Initialize environment
env = gym.make(game)
num_actions = env.action_space.n
# Initialize constants
num_frames = 4
max_episodes = 1000000
max_frames = 10000
gamma = 0.95
# Cold start
if not warm_start:
# Initialize model
model = Policy(input_channels=num_frames, num_actions=num_actions)
# Initialize statistics
running_reward = None
running_rewards = []
prior_eps = 0
# Warm start
if warm_start:
data_file = 'results/{}.p'.format(game)
try:
with open(data_file, 'rb') as f:
running_rewards = pickle.load(f)
running_reward = running_rewards[-1]
prior_eps = len(running_rewards)
model_file = 'saved_models/actor_critic_{}_ep_{}.p'.format(
game, prior_eps)
with open(model_file, 'rb') as f:
model = pickle.load(f)
except OSError:
print('Saved file not found. Creating new cold start model.')
model = Policy(input_channels=num_frames, num_actions=num_actions)
running_reward = None
running_rewards = []
prior_eps = 0
cuda = torch.cuda.is_available()
if cuda:
model = model.cuda()
optimizer = optim.RMSprop(model.parameters(), lr=1e-4,
weight_decay=0.1) #LSTM Change: lr = 1e-4
for ep in range(max_episodes):
# Anneal temperature from 2.0 down to 0.5 over 10000 episodes
model.temperature = max(0.5, 2.0 - 1.5 * ((ep + prior_eps) / 1.0e4))
state = env.reset()
state = preprocess_state(state)
state = np.stack([state] * num_frames)
# LSTM change - reset LSTM hidden units when episode begins
cx = Variable(torch.zeros(1, 256))
hx = Variable(torch.zeros(1, 256))
if cuda:
cx = cx.cuda()
hx = hx.cuda()
reward_sum = 0.0
for frame in range(max_frames):
# Select action
# LSTM Change: Need to cycle hx and cx thru select_action
action, log_prob, state_value, (hx, cx) = select_action(
model, state, (hx, cx), cuda)
model.saved_actions.append((log_prob, state_value))
# Perform step
next_state, reward, done, info = env.step(action)
# Add reward to reward buffer
model.rewards.append(reward)
reward_sum += reward
# Compute latest state
next_state = preprocess_state(next_state)
# Evict oldest diff add new diff to state
next_state = np.stack([next_state] * num_frames)
next_state[1:, :, :] = state[:-1, :, :]
state = next_state
if done:
break
# Compute/display statistics
if running_reward is None:
running_reward = reward_sum
else:
running_reward = running_reward * 0.99 + reward_sum * 0.01
running_rewards.append(running_reward)
verbose_str = 'Episode {} complete'.format(ep + prior_eps + 1)
verbose_str += '\tReward total:{}'.format(reward_sum)
verbose_str += '\tRunning mean: {:.4}'.format(running_reward)
sys.stdout.write('\r' + verbose_str)
sys.stdout.flush()
# Update model
finish_episode(model, optimizer, gamma, cuda)
if (ep + prior_eps + 1) % 500 == 0:
model_file = 'saved_models/actor_critic_{}_ep_{}.p'.format(
game, ep + prior_eps + 1)
data_file = 'results/{}.p'.format(game)
with open(model_file, 'wb') as f:
pickle.dump(model.cpu(), f)
if cuda:
model = model.cuda()
with open(data_file, 'wb') as f:
pickle.dump(running_rewards, f)