def __init__(self, actor_state_size, actor_action_size, critic_state_size, critic_action_size, **kwargs):
if 'filename' in kwargs.keys():
data= torch.load(kwargs['filename'])
self.config= data["config"]
self.scores= data["scores"]
elif 'config' in kwargs.keys():
self.config= kwargs['config']
data= {}
self.scores= []
else:
raise OSError('DDPG: no configuration parameter in class init')
self.actor_state_size = actor_state_size
self.actor_action_size = actor_action_size
self.critic_state_size = critic_state_size
self.critic_action_size = critic_action_size
memory_size = self.config.get("memory_size", 100000)
actor_lr = self.config.get("actor_lr", 1e-3)
critic_lr = self.config.get("critic_lr", 1e-3)
self.batch_size = self.config.get("batch_size", 256)
self.discount = self.config.get("discount", 0.9)
sigma = self.config.get("sigma", 0.2)
self.tau= self.config.get("tau", 0.001)
self.seed = self.config.get("seed", 0)
self.action_noise= self.config.get("action_noise", "No")
self.critic_l2_reg= self.config.get("critic_l2_reg", 0.0)
random.seed(self.seed)
torch.manual_seed(self.seed)
param_noise= False
if self.action_noise== "Param": param_noise= True
self.actor = Actor(actor_state_size, actor_action_size, nodes= self.config["actor_nodes"], seed= self.seed, param_noise= param_noise).to(device)
self.critic = Critic(critic_state_size, critic_action_size, nodes= self.config["critic_nodes"], seed= self.seed).to(device)
self.targetActor = Actor(actor_state_size, actor_action_size, nodes= self.config["actor_nodes"], seed= self.seed, param_noise= param_noise).to(device)
self.targetCritic = Critic(critic_state_size, critic_action_size, nodes= self.config["critic_nodes"], seed= self.seed).to(device)
# Initialize parameters
self.hard_update(self.actor, self.targetActor)
self.hard_update(self.critic, self.targetCritic)
self.actor_optimizer = optim.Adam(self.actor.parameters(), lr= actor_lr)
self.critic_optimizer = optim.Adam(self.critic.parameters(), lr= critic_lr, weight_decay= self.critic_l2_reg)
self.criticLoss = nn.MSELoss() #nn.SmoothL1Loss()
#self.criticLoss = nn.SmoothL1Loss()
#self.noise= None
self.noise = NoNoise()
if self.action_noise== "OU":
self.noise = OUNoise(np.zeros(actor_action_size), sigma= sigma)
elif self.action_noise== "No":
self.noise = NoNoise()
elif self.action_noise== "Normal":
self.noise = NormalActionNoise(np.zeros(actor_action_size), sigma= sigma)
self.memory = Memory(memory_size, self.batch_size, self.seed)
def __init__(self, **kwargs):
if 'filename' in kwargs.keys():
data= torch.load(kwargs['filename'])
self.config= data["config"]
self.scores= data["scores"]
elif 'config' in kwargs.keys():
self.config= kwargs['config']
data= {}
self.scores= []
else:
raise OSError('DDPG: no configuration parameter in class init')
self.state_size = self.config["state_size"]
self.action_size = self.config["action_size"]
memory_size = self.config["memory_size"]
actor_lr = self.config["actor_lr"]
critic_lr = self.config["critic_lr"]
self.batch_size = self.config["batch_size"]
self.discount = self.config["discount"]
sigma = self.config["sigma"] if self.config["sigma"] else 0.2
self.tau= self.config["tau"]
self.seed = self.config["seed"] if self.config["seed"] else 0
self.action_noise= self.config["action_noise"] if self.config["action_noise"] else "No"
self.critic_l2_reg= self.config["critic_l2_reg"] if self.config["critic_l2_reg"] else 0.0
random.seed(self.seed)
torch.manual_seed(self.seed)
self.actor = Actor(self.state_size, self.action_size, nodes= self.config["actor_nodes"], seed= self.seed).to(device)
if 'actor' in data.keys(): self.actor.load_state_dict(data['actor'])
self.critic = Critic(self.state_size, self.action_size, nodes= self.config["critic_nodes"], seed= self.seed).to(device)
self.targetActor = Actor(self.state_size, self.action_size, nodes= self.config["actor_nodes"], seed= self.seed).to(device)
self.targetCritic = Critic(self.state_size, self.action_size, nodes= self.config["critic_nodes"], seed= self.seed).to(device)
# Initialize parameters
self.hard_update(self.actor, self.targetActor)
self.hard_update(self.critic, self.targetCritic)
self.actor_optimizer = optim.Adam(self.actor.parameters(), lr= actor_lr)
self.critic_optimizer = optim.Adam(self.critic.parameters(), lr= critic_lr, weight_decay= self.critic_l2_reg)
self.criticLoss = nn.MSELoss()
self.noise= None
if self.action_noise== "OU":
self.noise = OUNoise(np.zeros(self.action_size), sigma= sigma)
elif self.action_noise== "No":
self.noise = NoNoise()
elif self.action_noise== "Normal":
self.noise = NormalActionNoise(np.zeros(self.action_size), sigma= sigma)
self.memory = ReplayBuffer(self.action_size, memory_size, self.batch_size, self.seed)
def eval_from_checkpoint(env, config, run=0):
"""
Apply procedures of training for a DDPG.
"""
experience = ReplayBuffer()
noise = NormalActionNoise(0, 0.1, size=env.action_space.shape[0])
# initialize
agent = TD3DDPG(env, config, experience, action_noise=noise, run=run)
agent.initialize()
agent.restore()
agent.record()
agent.close()
def ddpg(episode, breaking_step, reward_name):
env = gym.make('AntPyBulletEnv-v0')
cumulus_steps = 0
episode_steps = 0
# randomly initialize critics and actor with weights and biases
q1 = CriticNN()
q1.compile(optimizer=Adam(learning_rate=0.001), loss='mse')
q2 = CriticNN()
q2.compile(optimizer=Adam(learning_rate=0.001), loss='mse')
mu = ActorNN(env.action_space.shape[0])
mu.compile(optimizer=Adam(learning_rate=0.001), loss='mse')
# initialize target networks
q1_target = CriticNN()
q1_target.compile(optimizer=Adam(learning_rate=0.001), loss='mse')
q2_target = CriticNN()
q2_target.compile(optimizer=Adam(learning_rate=0.001), loss='mse')
mu_target = ActorNN(env.action_space.shape[0])
mu_target.compile(optimizer=Adam(learning_rate=0.001), loss='mse')
q1_target, q2_target, mu_target = update_network_parameters(
q1, q1_target, q2, q2_target, mu, mu_target, 0.005)
# initialize replay buffer (actor critic train only after batch is full 64!)
replay_buffer = ReplayBuffer(1000000, env.observation_space.shape[0],
env.action_space.shape[0])
performance = []
avg_return = []
time_step_reward = []
avg_time_step_reward = []
a_c = 0
b_c = 0
c_c = 0
d_c = 0
e_c = 0
f_c = 0
for e in range(episode):
# receive initial observation state s1 (observation = s1)
# env.render()
observation = env.reset()
state = tf.convert_to_tensor([observation], dtype=tf.float32)
max_steps = 1000
min_action = env.action_space.low[0]
max_action = env.action_space.high[0]
update_frequency = 2
learn_count = 0
score = 0
for i in range(max_steps):
# select an action a_t = mu(state) + noise
noise = NormalActionNoise(0, 0.1)
action = mu(state) + np.random.normal(noise.mean, noise.sigma)
proto_tensor = tf.make_tensor_proto(action)
action = tf.make_ndarray(proto_tensor)
action = action[0]
# execute action a_t and observe reward, and next state
next_state, reward, done, _ = env.step(action)
# store transition in replay buffer
replay_buffer.store_transition(state, action, reward, next_state,
done)
# if there are enough transitions in the replay buffer
batch_size = 100
if replay_buffer.mem_cntr >= batch_size:
# sample a random mini batch of n=64 transitions
buff_state, buff_action, buff_reward, buff_next_state, buff_done = replay_buffer.sample_buffer(
batch_size)
states = tf.convert_to_tensor(buff_state, dtype=tf.float32)
next_states = tf.convert_to_tensor(buff_next_state,
dtype=tf.float32)
rewards = tf.convert_to_tensor(buff_reward, dtype=tf.float32)
actions = tf.convert_to_tensor(buff_action, dtype=tf.float32)
# train critics
with tf.GradientTape(persistent=True) as tape:
# calculate which actions target_actor chooses and add noise
target_actions = mu_target(next_states) + tf.clip_by_value(
np.random.normal(scale=0.2), -0.5, 0.5)
target_actions = tf.clip_by_value(target_actions,
min_action, max_action)
# calculate next_q_values of the critic by feeding the next state and from actor chosen actions
next_critic_value1 = tf.squeeze(
q1_target(next_states, target_actions), 1)
next_critic_value2 = tf.squeeze(
q2_target(next_states, target_actions), 1)
# calculate q values of critic actual state
critic_value1 = tf.squeeze(q1(states, actions), 1)
critic_value2 = tf.squeeze(q2(states, actions), 1)
# use smaller q value from the 2 critics
next_critic_value = tf.math.minimum(
next_critic_value1, next_critic_value2)
# calculate target values: yt = rt + gamma * q_target(s_t+1, mu_target(s_t+1)); with t = time step
y = rewards + 0.99 * next_critic_value * (1 - buff_done)
# calculate the loss between critic and target_critic
critic1_loss = keras.losses.MSE(y, critic_value1)
critic2_loss = keras.losses.MSE(y, critic_value2)
# update critics by minimized the loss (critic_loss) and using Adam optimizer
critic1_network_gradient = tape.gradient(
critic1_loss, q1.trainable_variables)
critic2_network_gradient = tape.gradient(
critic2_loss, q2.trainable_variables)
q1.optimizer.apply_gradients(
zip(critic1_network_gradient, q1.trainable_variables))
q2.optimizer.apply_gradients(
zip(critic2_network_gradient, q2.trainable_variables))
learn_count += 1
# train actor
if learn_count % update_frequency == 0:
with tf.GradientTape() as tape:
new_policy_actions = mu(states)
# check if - or + (descent or ascent) not sure yet
actor_loss = -q1(states, new_policy_actions)
actor_loss = tf.math.reduce_mean(actor_loss)
# update the actor policy using the sampled policy gradient
actor_network_gradient = tape.gradient(
actor_loss, mu.trainable_variables)
mu.optimizer.apply_gradients(
zip(actor_network_gradient, mu.trainable_variables))
# update the target networks
update_network_parameters(q1, q1_target, q2, q2_target, mu,
mu_target, 0.005)
time_step_reward.append(reward)
avg_time_step_reward_short = np.mean(time_step_reward[-50:])
avg_time_step_reward.append(avg_time_step_reward_short)
if done:
performance.append(score)
avg_reward = np.mean(performance[-50:])
avg_return.append(avg_reward)
cumulus_steps += i
print(
"episode: {}/{}, score: {}, avg_score: {}, ep_steps: {}, cumulus_steps: {}"
.format(e, episode, score, avg_reward, i, cumulus_steps))
if 10000 < cumulus_steps < 11000 and a_c == 0:
a_c = 1
if not os.path.exists(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}"
.format(reward_name)):
os.mkdir(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}"
.format(reward_name))
mu.save_weights(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/mu{}.h5"
.format(reward_name, cumulus_steps))
np.save(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/avg_return{}"
.format(reward_name, cumulus_steps), avg_return)
np.save(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/time_step_reward{}"
.format(reward_name, cumulus_steps), time_step_reward)
np.save(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/performance{}"
.format(reward_name, cumulus_steps), performance)
np.save(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/avg_time_step_reward{}"
.format(reward_name,
cumulus_steps), avg_time_step_reward)
if 150000 < cumulus_steps < 151000 and b_c == 0:
b_c = 1
if not os.path.exists(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}"
.format(reward_name)):
os.mkdir(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}"
.format(reward_name))
mu.save_weights(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/mu{}.h5"
.format(reward_name, cumulus_steps))
np.save(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/avg_return{}"
.format(reward_name, cumulus_steps), avg_return)
np.save(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/time_step_reward{}"
.format(reward_name, cumulus_steps), time_step_reward)
np.save(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/performance{}"
.format(reward_name, cumulus_steps), performance)
np.save(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/avg_time_step_reward{}"
.format(reward_name,
cumulus_steps), avg_time_step_reward)
# if 350000 < cumulus_steps < 351000 and c_c == 0:
# c_c = 1
# if not os.path.exists("/home/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}".format(reward_name)):
# os.mkdir("/home/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}".format(reward_name))
# mu.save_weights("/home/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/mu{}.h5".format(reward_name, cumulus_steps))
# np.save("/home/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/avg_return{}".format(reward_name, cumulus_steps), avg_return)
# np.save("/home/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/time_step_reward{}".format(reward_name, cumulus_steps), time_step_reward)
# np.save("/home/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/performance{}".format(reward_name, cumulus_steps), performance)
# np.save("/home/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/avg_time_step_reward{}".format(reward_name, cumulus_steps), avg_time_step_reward)
if 550000 < cumulus_steps < 551000 and d_c == 0:
d_c = 1
if not os.path.exists(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}"
.format(reward_name)):
os.mkdir(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}"
.format(reward_name))
mu.save_weights(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/mu{}.h5"
.format(reward_name, cumulus_steps))
np.save(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/avg_return{}"
.format(reward_name, cumulus_steps), avg_return)
np.save(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/time_step_reward{}"
.format(reward_name, cumulus_steps), time_step_reward)
np.save(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/performance{}"
.format(reward_name, cumulus_steps), performance)
np.save(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/avg_time_step_reward{}"
.format(reward_name,
cumulus_steps), avg_time_step_reward)
if 750000 < cumulus_steps < 751000 and e_c == 0:
e_c = 1
if not os.path.exists(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}"
.format(reward_name)):
os.mkdir(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}"
.format(reward_name))
mu.save_weights(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/mu{}.h5"
.format(reward_name, cumulus_steps))
np.save(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/avg_return{}"
.format(reward_name, cumulus_steps), avg_return)
np.save(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/time_step_reward{}"
.format(reward_name, cumulus_steps), time_step_reward)
np.save(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/performance{}"
.format(reward_name, cumulus_steps), performance)
np.save(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/avg_time_step_reward{}"
.format(reward_name,
cumulus_steps), avg_time_step_reward)
if 1000000 < cumulus_steps < 1001000 and f_c == 0:
f_c = 1
mu.save_weights(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/mu{}.h5"
.format(reward_name, cumulus_steps))
np.save(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/avg_return{}"
.format(reward_name, cumulus_steps), avg_return)
np.save(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/time_step_reward{}"
.format(reward_name, cumulus_steps), time_step_reward)
np.save(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/performance{}"
.format(reward_name, cumulus_steps), performance)
np.save(
"/var/tmp/ga53cov/Bachelor_Arbeit/BA/Models/Ant_v2/{}/avg_time_step_reward{}"
.format(reward_name,
cumulus_steps), avg_time_step_reward)
break
score += reward
state = tf.convert_to_tensor([next_state], dtype=tf.float32)
# stop learning after certain time steps
if cumulus_steps > breaking_step:
break
return avg_return, mu, performance, time_step_reward, avg_time_step_reward
def learn(
env,
seed=None,
total_timesteps=None,
nb_epochs=None, # with default settings, perform 1M steps total
nb_epoch_cycles=20,
nb_rollout_steps=100,
reward_scale=1.0,
render=False,
render_eval=False,
noise_type='adaptive-param_0.2',
normalize_returns=False,
normalize_observations=True,
critic_l2_reg=1e-2,
actor_lr=1e-4,
critic_lr=1e-3,
popart=False,
gamma=0.99,
clip_norm=None,
nb_train_steps=50, # per epoch cycle and MPI worker,
nb_eval_steps=100,
nb_save_epochs=None,
batch_size=64, # per MPI worker
tau=0.01,
action_range=(-250.0, 250.0),
observation_range=(-5.0, 5.0),
eval_env=None,
load_path=None,
save_dir=None,
param_noise_adaption_interval=50,
**network_kwargs):
set_global_seeds(seed)
if total_timesteps is not None:
assert nb_epochs is None
nb_epochs = int(total_timesteps) // (nb_epoch_cycles *
nb_rollout_steps)
else:
nb_epochs = 500
if MPI is not None:
rank = MPI.COMM_WORLD.Get_rank()
else:
rank = 0
memory = Memory(limit=int(1e6))
network_spec = [{
'layer_type': 'dense',
'units': int(256),
'activation': 'relu',
'nodes_in': ['main'],
'nodes_out': ['main']
}, {
'layer_type': 'dense',
'units': int(128),
'activation': 'relu',
'nodes_in': ['main'],
'nodes_out': ['main']
}, {
'layer_type': 'dense',
'units': int(1),
'activation': 'tanh',
'nodes_in': ['main'],
'nodes_out': ['main']
}]
vnetwork_spec = [{
'layer_type': 'concat',
'nodes_in': ['action_movement', 'observation_self'],
'nodes_out': ['main']
}, {
'layer_type': 'dense',
'units': int(256),
'activation': 'relu',
'nodes_in': ['main'],
'nodes_out': ['main']
}, {
'layer_type': 'dense',
'units': int(128),
'activation': 'relu',
'nodes_in': ['main'],
'nodes_out': ['main']
}, {
'layer_type': 'dense',
'units': int(1),
'activation': '',
'nodes_in': ['main'],
'nodes_out': ['main']
}]
network = DdpgPolicy(scope="ddpg",
ob_space=env.observation_space,
ac_space=env.action_space,
network_spec=network_spec,
v_network_spec=vnetwork_spec,
stochastic=False,
reuse=False,
build_act=True,
trainable_vars=None,
not_trainable_vars=None,
gaussian_fixed_var=False,
weight_decay=0.0,
ema_beta=0.99999,
normalize_observations=normalize_observations,
normalize_returns=normalize_returns,
observation_range=observation_range)
target_network = DdpgPolicy(scope="target",
ob_space=env.observation_space,
ac_space=env.action_space,
network_spec=network_spec,
v_network_spec=vnetwork_spec,
stochastic=False,
reuse=False,
build_act=True,
trainable_vars=None,
not_trainable_vars=None,
gaussian_fixed_var=False,
weight_decay=0.0,
ema_beta=0.99999,
normalize_observations=normalize_observations,
normalize_returns=normalize_returns,
observation_range=observation_range)
action_noise = None
param_noise = None
if noise_type is not None:
for current_noise_type in noise_type.split(','):
current_noise_type = current_noise_type.strip()
if current_noise_type == 'none':
pass
elif 'adaptive-param' in current_noise_type:
_, stddev = current_noise_type.split('_')
param_noise = AdaptiveParamNoiseSpec(
initial_stddev=float(stddev),
desired_action_stddev=float(stddev))
elif 'normal' in current_noise_type:
action_noise = dict()
for k, v in env.action_space.spaces.items():
act_size = v.spaces[0].shape[-1]
_, stddev = current_noise_type.split('_')
action_noise[k] = NormalActionNoise(mu=np.zeros(act_size),
sigma=float(stddev) *
np.ones(act_size))
elif 'ou' in current_noise_type:
action_noise = dict()
for k, v in env.action_space.spaces.items():
act_size = v.spaces[0].shape[-1]
_, stddev = current_noise_type.split('_')
action_noise[k] = OrnsteinUhlenbeckActionNoise(
mu=np.zeros(act_size),
sigma=float(stddev) * np.ones(act_size))
else:
raise RuntimeError(
'unknown noise type "{}"'.format(current_noise_type))
max_action = action_range[1]
logger.info(
'scaling actions by {} before executing in env'.format(max_action))
agent = DDPG(network,
target_network,
memory,
env.observation_space,
env.action_space,
gamma=gamma,
tau=tau,
normalize_returns=normalize_returns,
normalize_observations=normalize_observations,
batch_size=batch_size,
action_noise=action_noise,
param_noise=param_noise,
critic_l2_reg=critic_l2_reg,
actor_lr=actor_lr,
critic_lr=critic_lr,
enable_popart=popart,
clip_norm=clip_norm,
reward_scale=reward_scale)
logger.info('Using agent with the following configuration:')
logger.info(str(agent.__dict__.items()))
eval_episode_rewards_history = deque(maxlen=100)
episode_rewards_history = deque(maxlen=100)
sess = U.get_session()
saver = functools.partial(save_variables, sess=sess)
loader = functools.partial(load_variables, sess=sess)
if load_path != None:
loader(load_path)
# Prepare everything.
agent.initialize(sess)
sess.graph.finalize()
agent.reset()
obs = env.reset()
if eval_env is not None:
eval_obs = eval_env.reset()
nenvs = env.num_envs
n_agents = obs['observation_self'].shape[0]
episode_reward = np.zeros((nenvs, n_agents), dtype=np.float32) #vector
episode_step = np.zeros(nenvs, dtype=int) # vector
episodes = 0 #scalar
t = 0 # scalar
epoch = 0
start_time = time.time()
epoch_episode_rewards = []
epoch_episode_steps = []
epoch_actions = []
epoch_qs = []
epoch_episodes = 0
for epoch in range(nb_epochs):
for cycle in range(nb_epoch_cycles):
# Perform rollouts.
if nenvs > 1:
# if simulating multiple envs in parallel, impossible to reset agent at the end of the episode in each
# of the environments, so resetting here instead
agent.reset()
for t_rollout in range(nb_rollout_steps):
# Predict next action.
action, q, _, _ = agent.step(obs,
apply_noise=True,
compute_Q=True)
# Execute next action.
if rank == 0 and render:
env.render()
# max_action is of dimension A, whereas action is dimension (nenvs, A) - the multiplication gets broadcasted to the batch
for k, v in action.items():
action[k] *= max_action
nenvs_actions = []
for i in range(nenvs):
nenv_action = {
'action_movement':
action['action_movement'][i * n_agents:(i + 1) *
n_agents]
}
nenvs_actions.append(nenv_action)
new_obs, r, done, info = env.step(
nenvs_actions
) # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1])
# note these outputs are batched from vecenv
t += 1
if rank == 0 and render:
env.render()
episode_reward += r
episode_step += 1
# Book-keeping.
epoch_actions.append(action)
epoch_qs.append(q)
agent.store_transition(
obs, action, r, new_obs, done
) #the batched data will be unrolled in memory.py's append.
obs = new_obs
for d in range(len(done)):
if done[d]:
# Episode done.
epoch_episode_rewards.append(episode_reward[d])
episode_rewards_history.append(episode_reward[d])
epoch_episode_steps.append(episode_step[d])
episode_reward[d] = 0.
episode_step[d] = 0
epoch_episodes += 1
episodes += 1
if nenvs == 1:
agent.reset()
# Train.
epoch_actor_losses = []
epoch_critic_losses = []
epoch_adaptive_distances = []
for t_train in range(nb_train_steps):
# Adapt param noise, if necessary.
if memory.nb_entries >= batch_size and t_train % param_noise_adaption_interval == 0:
distance = agent.adapt_param_noise()
epoch_adaptive_distances.append(distance)
cl, al = agent.train()
epoch_critic_losses.append(cl)
epoch_actor_losses.append(al)
agent.update_target_net()
# Evaluate.
eval_episode_rewards = []
eval_qs = []
if eval_env is not None:
nenvs_eval = eval_obs.shape[0]
eval_episode_reward = np.zeros(nenvs_eval, dtype=np.float32)
for t_rollout in range(nb_eval_steps):
eval_action, eval_q, _, _ = agent.step(eval_obs,
apply_noise=False,
compute_Q=True)
eval_obs, eval_r, eval_done, eval_info = eval_env.step(
max_action * eval_action
) # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1])
if render_eval:
eval_env.render()
eval_episode_reward += eval_r
eval_qs.append(eval_q)
for d in range(len(eval_done)):
if eval_done[d]:
eval_episode_rewards.append(eval_episode_reward[d])
eval_episode_rewards_history.append(
eval_episode_reward[d])
eval_episode_reward[d] = 0.0
if MPI is not None:
mpi_size = MPI.COMM_WORLD.Get_size()
else:
mpi_size = 1
# Log stats.
# XXX shouldn't call np.mean on variable length lists
duration = time.time() - start_time
stats = agent.get_stats()
combined_stats = stats.copy()
combined_stats['rollout/return'] = np.mean(epoch_episode_rewards)
combined_stats['rollout/return_std'] = np.std(epoch_episode_rewards)
combined_stats['rollout/return_history'] = np.mean(
episode_rewards_history)
combined_stats['rollout/return_history_std'] = np.std(
episode_rewards_history)
combined_stats['rollout/episode_steps'] = np.mean(epoch_episode_steps)
combined_stats['rollout/Q_mean'] = np.mean(epoch_qs)
combined_stats['train/loss_actor'] = np.mean(epoch_actor_losses)
combined_stats['train/loss_critic'] = np.mean(epoch_critic_losses)
combined_stats['train/param_noise_distance'] = np.mean(
epoch_adaptive_distances)
combined_stats['total/duration'] = duration
combined_stats['total/steps_per_second'] = float(t) / float(duration)
combined_stats['total/episodes'] = episodes
combined_stats['rollout/episodes'] = epoch_episodes
# Evaluation statistics.
if eval_env is not None:
combined_stats['eval/return'] = eval_episode_rewards
combined_stats['eval/return_history'] = np.mean(
eval_episode_rewards_history)
combined_stats['eval/Q'] = eval_qs
combined_stats['eval/episodes'] = len(eval_episode_rewards)
def as_scalar(x):
if isinstance(x, np.ndarray):
assert x.size == 1
return x[0]
elif np.isscalar(x):
return x
else:
raise ValueError('expected scalar, got %s' % x)
combined_stats_sums = np.array(
[np.array(x).flatten()[0] for x in combined_stats.values()])
if MPI is not None:
combined_stats_sums = MPI.COMM_WORLD.allreduce(combined_stats_sums)
combined_stats = {
k: v / mpi_size
for (k, v) in zip(combined_stats.keys(), combined_stats_sums)
}
# Total statistics.
combined_stats['total/epochs'] = epoch + 1
combined_stats['total/steps'] = t
for key in sorted(combined_stats.keys()):
logger.record_tabular(key, combined_stats[key])
if rank == 0:
logger.dump_tabular()
logger.info('')
logdir = logger.get_dir()
if rank == 0 and logdir:
if hasattr(env, 'get_state'):
with open(os.path.join(logdir, 'env_state.pkl'), 'wb') as f:
pickle.dump(env.get_state(), f)
if eval_env and hasattr(eval_env, 'get_state'):
with open(os.path.join(logdir, 'eval_env_state.pkl'),
'wb') as f:
pickle.dump(eval_env.get_state(), f)
if nb_save_epochs != None and (epoch + 1) % nb_save_epochs == 0:
if save_dir == None:
checkdir = osp.join(logger.get_dir(), 'checkpoints')
else:
checkdir = osp.join(save_dir, 'checkpoints')
os.makedirs(checkdir, exist_ok=True)
savepath = osp.join(checkdir, '%.5i' % epoch)
print('Saving to', savepath)
saver(savepath)
return agent
def learn(env,
config,
num_episodes=5000,
num_eval_final=50,
batch_size=100,
seed=7,
run=0,
record=False):
"""
Apply procedures of training for a DDPG.
"""
experience = ReplayBuffer()
noise = NormalActionNoise(0, 0.1, size=env.action_space.shape[0])
config.batch_size = batch_size
# initialize
agent = TD3DDPG(env, config, experience, action_noise=noise, run=run)
agent.initialize()
# record one game at the beginning
# if agent.config.record:
# agent.record()
# model
# Evaluate untrained policy
agent.evaluate_policy()
total_timesteps = 0
timesteps_since_eval = 0
episode_num = 0
done = True
stats = {}
stats["episode_rewards"] = []
stats["evaluation_rewards"] = []
stats["grad_norms"] = [] # TODO
stats["cummulative_timesteps"] = []
stats["episode_timesteps"] = []
stats["eval_episode_timesteps"] = []
stats["eval_cummulative_timesteps"] = []
stats["num_episodes"] = num_episodes
stats["num_eval_final"] = num_eval_final
stats["seed"] = seed
stats["agent"] = agent.agent_name
stats["env"] = config.env_name
while episode_num < num_episodes: #total_timesteps < config.max_timesteps:
if done:
if total_timesteps != 0:
print("Total T: {} Episode Num: {} Episode T: {} Reward: {}".
format(total_timesteps, episode_num, episode_timesteps,
episode_reward))
stats = agent.train(episode_timesteps, stats=stats)
stats["episode_rewards"].append(episode_reward)
stats["cummulative_timesteps"].append(total_timesteps)
stats["episode_timesteps"].append(episode_timesteps)
# # Evaluate episode
# if timesteps_since_eval >= config.eval_freq:
# timesteps_since_eval %= config.eval_freq
# agent.evaluate_policy()
# Reset environment
observation = env.reset()
done = False
episode_reward = 0
episode_timesteps = 0
episode_num += 1
# Select action randomly or according to policy
if total_timesteps < config.start_timesteps:
action = env.action_space.sample()
else:
action, _ = agent.act(np.array(observation),
apply_noise=True,
compute_q=False)
action = np.squeeze(action, axis=1)
# Perform action
new_observation, reward, done, _ = env.step(action)
done_bool = False if episode_timesteps + 1 == env._max_episode_steps else done
episode_reward += reward
# Store data in replay buffer
agent.add_experience(observation, action, reward, new_observation,
done_bool)
observation = new_observation
episode_timesteps += 1
total_timesteps += 1
timesteps_since_eval += 1
# Final evaluation
rewards, eval_cummulative_timesteps, eval_episode_timesteps = agent.evaluate_policy(
eval_episodes=num_eval_final)
stats["evaluation_rewards"] = rewards
stats["eval_cummulative_timesteps"] = eval_cummulative_timesteps
stats["eval_episode_timesteps"] = eval_episode_timesteps
agent.save_model()
if record:
agent.record()
agent.close()
return stats
def run(env_id, seed, noise_type, layer_norm, evaluation, memory_limit,
**kwargs):
rank = MPI.COMM_WORLD.Get_rank()
if rank != 0:
logger.set_level(logger.DISABLED)
print("rank: %d" % (rank))
env = gym.make(env_id)
env = bench.Monitor(
env,
logger.get_dir() and os.path.join(logger.get_dir(), str(rank)))
if evaluation and rank == 0:
eval_env = gym.make(env_id)
enal_env = bench.Monitor(eval_env,
os.path.join(logger.get_dir(), "gym_eval"))
env = bench.Monitor(env, None)
else:
eval_env = None
action_noise = None
param_noise = None
nb_actions = env.action_space.shape[-1]
for current_noise_type in noise_type.split(","):
current_noise_type = current_noise_type.strip()
if current_noise_type == "none":
pass
elif "adaptive-param" in current_noise_type:
_, stddev = current_noise_type.split("_")
param_noise = AdaptiveParamNoiseSpec(
initial_stddev=float(stddev),
desired_action_stddev=float(stddev))
elif "normal" in current_noise_type:
_, stddev = current_noise_type.split("_")
action_noise = NormalActionNoise(mu=np.zeros(nb_actions),
sigma=float(stddev) *
np.ones(nb_actions))
elif "ou" in current_noise_type.split("_"):
_, stddev = current_noise_type.split("_")
action_noise = OrnsteinUhlenbeckActionNoise(
mu=np.zeros(nb_actions),
sigma=float(stddev) * np.ones(nb_actions))
else:
raise RuntimeError(
'unknown noise type "{}"'.format(current_noise_type))
print(type(memory_limit), memory_limit)
memory = Memory(limit=int(memory_limit),
action_shape=env.action_space.shape,
observation_shape=env.observation_space.shape)
critic = Critic(layer_norm=layer_norm)
actor = Actor(nb_actions, layer_norm=layer_norm)
seed = seed + 1000000 * rank
logger.info("rank {} : seed={}, logdir={}".format(rank, seed,
logger.get_dir()))
tf.reset_default_graph()
set_global_seeds(seed)
env.seed(seed)
if eval_env is not None:
eval_env.seed(seed)
if rank == 0:
start_time = time.time()
if option == 1:
training.train(env=env,
eval_env=eval_env,
param_noise=param_noise,
action_noise=action_noise,
actor=actor,
critic=critic,
memory=memory,
**kwargs)
elif option == 2:
training_reward_shaping.train(env=env,
eval_env=eval_env,
param_noise=param_noise,
action_noise=action_noise,
actor=actor,
critic=critic,
memory=memory,
**kwargs)
env.close()
if eval_env is not None:
eval_env.close()
if rank == 0:
logger.info("total runtime: {}s".format(time.time() - start_time))
def learn(
env,
seed=None,
total_timesteps=1e6,
nb_epochs=None, # with default settings, perform 1M steps total
nb_rollout_steps=100,
max_ep_len=250,
reward_scale=1.0,
render=False,
render_eval=False,
noise_type='adaptive-param_0.2',
normalize_returns=False,
normalize_observations=True,
critic_l2_reg=1e-2,
actor_lr=1e-4,
critic_lr=1e-3,
popart=False,
gamma=0.99,
clip_norm=None,
start_steps=10000,
nb_train_steps=50, # per epoch cycle and MPI worker,
nb_eval_steps=100,
nb_log_steps=None,
nb_save_steps=None,
batch_size=64, # per MPI worker
polyak=0.01,
action_range=(-250.0, 250.0),
observation_range=(-5.0, 5.0),
target_noise=0.2,
noise_clip=0.5,
policy_delay=2,
eval_env=None,
load_path=None,
save_dir=None,
**network_kwargs):
set_global_seeds(seed)
if MPI is not None:
rank = MPI.COMM_WORLD.Get_rank()
else:
rank = 0
memory = Memory(limit=int(1e6))
network_spec = [{
'layer_type': 'dense',
'units': int(256),
'activation': 'relu',
'nodes_in': ['main'],
'nodes_out': ['main']
}, {
'layer_type': 'dense',
'units': int(128),
'activation': 'relu',
'nodes_in': ['main'],
'nodes_out': ['main']
}, {
'layer_type': 'dense',
'units': int(1),
'activation': 'tanh',
'nodes_in': ['main'],
'nodes_out': ['main']
}]
vnetwork_spec = [{
'layer_type': 'concat',
'nodes_in': ['action_movement', 'observation_self'],
'nodes_out': ['main']
}, {
'layer_type': 'dense',
'units': int(256),
'activation': 'relu',
'nodes_in': ['main'],
'nodes_out': ['main']
}, {
'layer_type': 'dense',
'units': int(128),
'activation': 'relu',
'nodes_in': ['main'],
'nodes_out': ['main']
}, {
'layer_type': 'dense',
'units': int(1),
'activation': '',
'nodes_in': ['main'],
'nodes_out': ['main']
}]
network = Td3Policy(scope="td3",
ob_space=env.observation_space,
ac_space=env.action_space,
network_spec=network_spec,
v_network_spec=vnetwork_spec,
stochastic=False,
reuse=False,
build_act=True,
trainable_vars=None,
not_trainable_vars=None,
gaussian_fixed_var=False,
weight_decay=0.0,
ema_beta=0.99999,
normalize_observations=normalize_observations,
normalize_returns=normalize_returns,
observation_range=observation_range,
action_range=action_range,
target_noise=target_noise,
noise_clip=noise_clip)
target_network = Td3Policy(scope="target",
ob_space=env.observation_space,
ac_space=env.action_space,
network_spec=network_spec,
v_network_spec=vnetwork_spec,
stochastic=False,
reuse=False,
build_act=True,
trainable_vars=None,
not_trainable_vars=None,
gaussian_fixed_var=False,
weight_decay=0.0,
ema_beta=0.99999,
normalize_observations=normalize_observations,
normalize_returns=normalize_returns,
observation_range=observation_range,
action_range=action_range,
target_noise=target_noise,
noise_clip=noise_clip,
isTarget=True)
action_noise = None
param_noise = None
if noise_type is not None:
for current_noise_type in noise_type.split(','):
current_noise_type = current_noise_type.strip()
if current_noise_type == 'none':
pass
elif 'adaptive-param' in current_noise_type:
_, stddev = current_noise_type.split('_')
param_noise = AdaptiveParamNoiseSpec(
initial_stddev=float(stddev),
desired_action_stddev=float(stddev))
elif 'normal' in current_noise_type:
action_noise = dict()
for k, v in env.action_space.spaces.items():
act_size = v.spaces[0].shape[-1]
_, stddev = current_noise_type.split('_')
action_noise[k] = NormalActionNoise(mu=np.zeros(act_size),
sigma=float(stddev) *
np.ones(act_size))
elif 'ou' in current_noise_type:
action_noise = dict()
for k, v in env.action_space.spaces.items():
act_size = v.spaces[0].shape[-1]
_, stddev = current_noise_type.split('_')
action_noise[k] = OrnsteinUhlenbeckActionNoise(
mu=np.zeros(act_size),
sigma=float(stddev) * np.ones(act_size))
else:
raise RuntimeError(
'unknown noise type "{}"'.format(current_noise_type))
max_action = action_range[1]
logger.info(
'scaling actions by {} before executing in env'.format(max_action))
agent = TD3(env,
network,
target_network,
memory,
env.action_space,
env.observation_space,
steps_per_epoch=nb_rollout_steps,
epochs=nb_epochs,
gamma=gamma,
polyak=polyak,
actor_lr=actor_lr,
critic_lr=critic_lr,
batch_size=batch_size,
start_steps=start_steps,
action_noise=action_noise,
target_noise=target_noise,
noise_clip=noise_clip,
policy_delay=policy_delay)
logger.info('Using agent with the following configuration:')
logger.info(str(agent.__dict__.items()))
eval_episode_rewards_history = deque(maxlen=100)
episode_rewards_history = deque(maxlen=100)
sess = U.get_session()
saver = functools.partial(save_variables, sess=sess)
loader = functools.partial(load_variables, sess=sess)
if load_path != None:
loader(load_path)
# Prepare everything.
agent.initialize(sess)
sess.graph.finalize()
agent.reset()
obs = env.reset()
if eval_env is not None:
eval_obs = eval_env.reset()
nenvs = env.num_envs
n_agents = obs['observation_self'].shape[0]
episode_reward = np.zeros((nenvs, n_agents), dtype=np.float32) #vector
episode_step = np.zeros(nenvs, dtype=int) # vector
episodes = 0 #scalar
t = 0 # scalar
epoch = 0
start_time = time.time()
epoch_episode_rewards = []
epoch_episode_steps = []
epoch_actions = []
epoch_qs = []
epoch_episodes = 0
for t in range(int(total_timesteps)):
"""
Until start_steps have elapsed, randomly sample actions
from a uniform distribution for better exploration. Afterwards,
use the learned policy (with some noise, via act_noise).
"""
if t > start_steps:
action, q, _, _ = agent.step(obs, apply_noise=True, compute_Q=True)
nenvs_actions = []
for i in range(nenvs):
nenv_action = {
'action_movement':
action['action_movement'][i * n_agents:(i + 1) * n_agents]
}
nenvs_actions.append(nenv_action)
else:
action, q = env.action_space.sample(), None
nenvs_actions = []
for i in range(nenvs):
nenv_action = {
'action_movement':
action['action_movement'][i * n_agents:(i + 1) *
n_agents][0]
}
nenvs_actions.append(nenv_action)
new_obs, r, done, info = env.step(nenvs_actions)
episode_reward += r
episode_step += 1
for d in range(len(done)):
done[d] = False if episode_step == max_ep_len else done[d]
epoch_actions.append(action)
epoch_qs.append(q)
agent.store_transition(
obs, action, r, new_obs,
done) #the batched data will be unrolled in memory.py's append.
obs = new_obs
for d in range(len(done)):
if done[d]:
# Episode done.
epoch_episode_rewards.append(episode_reward[d])
episode_rewards_history.append(episode_reward[d])
epoch_episode_steps.append(episode_step[d])
episode_reward[d] = 0.
episode_step[d] = 0
epoch_episodes += 1
episodes += 1
if nenvs == 1:
agent.reset()
episode_actor_losses = []
episode_critic_losses = []
episode_critic = []
episode_critic_twin = []
if d or (episode_step[0] == max_ep_len):
"""
Perform all TD3 updates at the end of the trajectory
(in accordance with source code of TD3 published by
original authors).
"""
for j in range(episode_step[0]):
critic_loss, critic, critic_twin, actor_loss = agent.train(
episode_step[0])
episode_critic_losses.append(critic_loss)
episode_critic.append(critic)
episode_critic_twin.append(critic_twin)
if actor_loss is not None:
episode_actor_losses.append(actor_loss)
obs, r, done, episode_reward, episode_step = env.reset(
), 0, False, np.zeros((nenvs, n_agents),
dtype=np.float32), np.zeros(nenvs, dtype=int)
if (t + 1) % nb_log_steps == 0:
# Log stats.
# XXX shouldn't call np.mean on variable length lists
duration = time.time() - start_time
stats = agent.get_stats()
combined_stats = stats.copy()
combined_stats['rollout/return'] = np.mean(epoch_episode_rewards)
combined_stats['rollout/return_std'] = np.std(
epoch_episode_rewards)
combined_stats['rollout/return_history'] = np.mean(
episode_rewards_history)
combined_stats['rollout/return_history_std'] = np.std(
episode_rewards_history)
combined_stats['rollout/episode_steps'] = np.mean(
epoch_episode_steps)
combined_stats['train/loss_actor'] = np.mean(episode_actor_losses)
combined_stats['train/loss_critic'] = np.mean(
episode_critic_losses)
combined_stats['total/duration'] = duration
combined_stats['total/steps_per_second'] = float(t) / float(
duration)
combined_stats['total/episodes'] = episodes
combined_stats['rollout/episodes'] = epoch_episodes
def as_scalar(x):
if isinstance(x, np.ndarray):
assert x.size == 1
return x[0]
elif np.isscalar(x):
return x
else:
raise ValueError('expected scalar, got %s' % x)
combined_stats_sums = np.array(
[np.array(x).flatten()[0] for x in combined_stats.values()])
# Total statistics.
combined_stats['total/epochs'] = epoch + 1
combined_stats['total/steps'] = t
for key in sorted(combined_stats.keys()):
logger.record_tabular(key, combined_stats[key])
if rank == 0:
logger.dump_tabular()
logger.info('')
logdir = logger.get_dir()
if rank == 0 and logdir:
if hasattr(env, 'get_state'):
with open(os.path.join(logdir, 'env_state.pkl'),
'wb') as f:
pickle.dump(env.get_state(), f)
if eval_env and hasattr(eval_env, 'get_state'):
with open(os.path.join(logdir, 'eval_env_state.pkl'),
'wb') as f:
pickle.dump(eval_env.get_state(), f)
if nb_save_steps != None and (t + 1) % nb_save_steps == 0:
if save_dir == None:
checkdir = osp.join(logger.get_dir(), 'checkpoints')
else:
checkdir = osp.join(save_dir, 'checkpoints')
os.makedirs(checkdir, exist_ok=True)
savepath = osp.join(checkdir, '%.5i' % t)
print('Saving to', savepath)
saver(savepath)
return agent
def __init__(self, env, args):
ob_space = env.observation_space
goal_dim = env.goal_dim
ob_dim = ob_space.shape[0]
self.ob_dim = ob_dim
self.ac_dim = ac_dim = 7
self.goal_dim = goal_dim
self.num_iters = args.num_iters
self.random_prob = args.random_prob
self.tau = args.tau
self.reward_scale = args.reward_scale
self.gamma = args.gamma
self.log_interval = args.log_interval
self.save_interval = args.save_interval
self.rollout_steps = args.rollout_steps
self.env = env
self.batch_size = args.batch_size
self.train_steps = args.train_steps
self.closest_dist = np.inf
self.warmup_iter = args.warmup_iter
self.max_grad_norm = args.max_grad_norm
self.use_her = args.her
self.k_future = args.k_future
self.model_dir = os.path.join(args.save_dir, 'model')
self.pretrain_dir = args.pretrain_dir
os.makedirs(self.model_dir, exist_ok=True)
self.global_step = 0
self.actor = Actor(ob_dim=ob_dim,
act_dim=ac_dim,
hid1_dim=args.hid1_dim,
hid2_dim=args.hid2_dim,
hid3_dim=args.hid3_dim,
init_method=args.init_method)
self.critic = Critic(ob_dim=ob_dim,
act_dim=ac_dim,
hid1_dim=args.hid1_dim,
hid2_dim=args.hid2_dim,
hid3_dim=args.hid3_dim,
init_method=args.init_method)
if args.resume or args.test or args.pretrain_dir is not None:
self.load_model(args.resume_step, pretrain_dir=args.pretrain_dir)
if not args.test:
self.actor_target = Actor(ob_dim=ob_dim,
act_dim=ac_dim,
hid1_dim=args.hid1_dim,
hid2_dim=args.hid2_dim,
hid3_dim=args.hid3_dim,
init_method=args.init_method)
self.critic_target = Critic(ob_dim=ob_dim,
act_dim=ac_dim,
hid1_dim=args.hid1_dim,
hid2_dim=args.hid2_dim,
hid3_dim=args.hid3_dim,
init_method=args.init_method)
self.actor_optim = self.construct_optim(self.actor,
lr=args.actor_lr)
cri_w_decay = args.critic_weight_decay
self.critic_optim = self.construct_optim(self.critic,
lr=args.critic_lr,
weight_decay=cri_w_decay)
self.hard_update(self.actor_target, self.actor)
self.hard_update(self.critic_target, self.critic)
self.actor_target.eval()
self.critic_target.eval()
if args.noise_type == 'ou_noise':
mu = np.zeros(ac_dim)
sigma = float(args.ou_noise_std) * np.ones(ac_dim)
self.action_noise = OrnsteinUhlenbeckActionNoise(mu=mu,
sigma=sigma)
elif args.noise_type == 'uniform':
low_limit = args.uniform_noise_low
high_limit = args.uniform_noise_high
dec_step = args.max_noise_dec_step
self.action_noise = UniformNoise(low_limit=low_limit,
high_limit=high_limit,
dec_step=dec_step)
elif args.noise_type == 'gaussian':
mu = np.zeros(ac_dim)
sigma = args.normal_noise_std * np.ones(ac_dim)
self.action_noise = NormalActionNoise(mu=mu, sigma=sigma)
self.memory = Memory(limit=int(args.memory_limit),
action_shape=(int(ac_dim), ),
observation_shape=(int(ob_dim), ))
self.critic_loss = nn.MSELoss()
self.ob_norm = args.ob_norm
if self.ob_norm:
self.obs_oms = OnlineMeanStd(shape=(1, ob_dim))
else:
self.obs_oms = None
self.cuda()
eval_env = gym.make(args.env)
#eval_env.seed(args.seed+100)
if logger.get_dir():
eval_env = bench.Monitor(
eval_env, os.path.join(logger.get_dir(), "eval.monitor.json"))
max_timesteps = train_env.spec.timestep_limit
# set noise type
current_noise_type = args.noise_type.strip()
nb_actions = train_env.action_space.shape[0]
if 'normal' in current_noise_type:
_, stddev = current_noise_type.split('_')
action_noise = NormalActionNoise(mu=np.zeros(nb_actions),
sigma=float(stddev) *
np.ones(nb_actions))
action_noise.reset()
if 'ou' in current_noise_type:
_, stddev = current_noise_type.split('_')
action_noise = OrnsteinUhlenbeckActionNoise(mu=np.zeros(nb_actions),
sigma=float(stddev) *
np.ones(nb_actions))
action_noise.reset()
else:
raise RuntimeError(
'unknown noise type "{}"'.format(current_noise_type))
episode_rewards = []
if 'Sparse' in train_env.spec.id:
sparse = True