class off_policy_parallel_rollouts():
def __init__(self, agent, env=None):
self.lock = th.Lock()
self.agent = agent
# one and only
self.farmer = farmer_class(self.agent.para_list)
self.ep_num = 0
self.total_steps = 0
self.history_reward = []
self.ep_value = {}
self.value_init()
self.relative_time = 0
self.average_steps = self.agent.para_list["max_pathlength"]
self.log_scalar_name_list = [
'mean_reward', 'actor_loss', 'critic_loss'
]
self.log_scalar_type_list = [tf.float32, tf.float32, tf.float32]
self.logger = Logger(self.agent.session,
self.agent.para_list["log_path"] + 'train',
self.log_scalar_name_list,
self.log_scalar_type_list)
self.write_log = self.logger.create_scalar_log_method()
self.start_time = time.time()
def refarm(self): # most time no use
del self.farmer
self.farmer = farmer_class()
def value_init(self):
self.ep_value['actor_loss'], self.ep_value[
'critic_loss'], self.ep_value['fit_time'] = 0, 0, 0
self.ep_value['sample_time'], self.ep_value['actor_train_time'] = 0, 0
self.ep_value['critic_train_time'], self.ep_value[
'target_update_time'] = 0, 0
def fit(self):
vars = [0] * 7
for i in range(self.average_steps):
var = self.agent.fit()
for i, v in enumerate(var):
vars[i] += v / self.average_steps
self.ep_value['actor_loss'], self.ep_value[
'critic_loss'], self.ep_value['fit_time'] = vars[0], vars[1], vars[
2]
self.ep_value['sample_time'], self.ep_value['actor_train_time'] = vars[
3], vars[4]
self.ep_value['critic_train_time'], self.ep_value[
'target_update_time'] = vars[5], vars[6]
def process_path(self, path):
ep_step = len(path["rewards"])
ep_time = path["ep_time"]
step_time = path["step_time"]
ep_reward = sum(path["rewards"])
ep_memory = []
for i in range(ep_step):
t = [
path["obs"][i], path["actions"][i], path["rewards"][i],
path["dones"][i], path["obs"][i + 1]
]
ep_memory.append(t)
return ep_step, ep_reward, ep_time, step_time, ep_memory
def rollout_an_episode(self, noise_level,
env): # this function is tread safe
global graph
with graph.as_default():
path = rollout_an_episode(self.agent, env, self.lock, noise_level)
self.fit()
ep_step, ep_reward, ep_time, step_time, ep_memory = self.process_path(
path)
for t in ep_memory:
self.agent.feed_one(t)
self.lock.acquire()
self.ep_num += 1
self.total_steps += ep_step # self.ep_value["episode_steps"]
self.average_steps = int(self.total_steps / self.ep_num)
self.ep_value["average_steps"] = self.average_steps
self.ep_value['episode_steps'], self.ep_value[
'episode_reward'] = ep_step, ep_reward
self.ep_value['episode_time'], self.ep_value[
'step_time'] = ep_time, step_time
self.history_reward.append(
ep_reward) # (self.ep_value["episode_reward"])
self.relative_time = (time.time() - self.start_time) / 60.0
off_policy_episode_print(self.ep_num, self.total_steps,
self.relative_time, self.ep_value)
self.write_log([
ep_reward, self.ep_value["actor_loss"],
self.ep_value["critic_loss"]
], self.ep_num)
self.lock.release()
env.rel()
def rollout_if_available(self, noise_level):
while True:
remote_env = self.farmer.acq_env() # call for a remote_env
if remote_env is False: # no free environment
# time.sleep(0.1)
pass
else:
t = th.Thread(target=self.rollout_an_episode,
args=(noise_level, remote_env),
daemon=True)
t.start()
break
def rollout(self):
for i in range(self.agent.para_list["n_episodes"]):
nl = noise_level_schedule(self.agent.para_list, self.total_steps)
self.rollout_if_available(nl)
if (i + 1) % 100 == 0:
self.agent.saveModel(0)
if self.total_steps > self.agent.para_list["n_steps"]:
break
self.agent.saveModel(0)
return self.history_reward
class parallel_rollouts():
def __init__(self, agent, env=None):
self.lock = th.Lock()
self.agent = agent
# one and only
self.para_list = self.get_parameter_list()
self.farmer = farmer_class(self.para_list)
self.ep_num = 0
self.total_steps = 0
self.history_reward = []
self.ep_value = {}
self.value_init()
self.relative_time = 0
self.average_len_of_episode = self.agent.args.max_pathlength
self.num_rollouts = int(self.agent.args.timesteps_per_batch /
self.average_len_of_episode)
self.rollout_count = 0
self.rollout_paths = []
self.iteration = 0
self.log_scalar_name_list = [
'reward', 'kl_div', 'entropy', 'surrogate_loss', 'value_loss'
]
self.log_scalar_type_list = [
tf.float32, tf.float32, tf.float32, tf.float32, tf.float32
]
self.logger = Logger(self.agent.session,
self.agent.args.log_path + 'train',
self.log_scalar_name_list,
self.log_scalar_type_list)
self.write_log = self.logger.create_scalar_log_method()
self.start_time = time.time()
def get_parameter_list(self):
para_list = {
"model": self.agent.args.model,
"task": self.agent.args.task,
"policy_layer_norm": self.agent.args.policy_layer_norm,
"value_layer_norm": self.agent.args.value_layer_norm,
"policy_act_fn": self.agent.args.policy_act_fn,
"value_act_fn": self.agent.args.value_act_fn,
"max_pathlength": self.agent.args.max_pathlength,
"farmer_port": self.agent.args.farmer_port,
"farm_list_base": self.agent.args.farm_list_base,
"farmer_debug_print": self.agent.args.farmer_debug_print,
"farm_debug_print": self.agent.args.farm_debug_print
}
return para_list
def refarm(self): # most time no use
del self.farmer
self.farmer = farmer_class()
def value_init(self):
# self.ep_value['actor_loss'], self.ep_value['critic_loss'], self.ep_value['fit_time'] = 0, 0, 0
# self.ep_value['sample_time'], self.ep_value['actor_train_time'] = 0, 0
# self.ep_value['critic_train_time'], self.ep_value['target_update_time'] = 0, 0
pass
def fit(self):
# calculate real leaning rate with a linear schedule
final_learning_rate = 5e-5
policy_delta_rate = (self.agent.args.policy_learning_rate -
final_learning_rate) / self.agent.args.n_steps
value_delta_rate = (self.agent.args.value_learning_rate -
final_learning_rate) / self.agent.args.n_steps
policy_learning_rate = self.agent.args.policy_learning_rate - policy_delta_rate * self.total_steps
value_learning_rate = self.agent.args.policy_learning_rate - value_delta_rate * self.total_steps
self.ep_value["learning_rate"] = policy_learning_rate
# vars: update_old_policy_time, train_time, fit_time, surrogate_loss, kl_after, entropy_after, value_loss
vars = self.agent.fit(self.rollout_paths,
[policy_learning_rate, value_learning_rate])
# update print info:
self.ep_value["episode_nums"] = self.num_rollouts
self.ep_value["episode_steps"] = sum(
[len(path["rewards"]) for path in self.rollout_paths])
self.ep_value["episode_time"] = sum(
[path["ep_time"]
for path in self.rollout_paths]) / self.num_rollouts
self.ep_value["episode_reward"] = sum(
[path["rewards"].sum()
for path in self.rollout_paths]) / self.num_rollouts
self.ep_value["average_steps"] = int(self.average_len_of_episode)
self.ep_value["step_time"] = sum(
[path["step_time"]
for path in self.rollout_paths]) / self.num_rollouts
self.ep_value['update_time'], self.ep_value[
'train_time'], self.ep_value['iter_time'] = vars[0], vars[1], vars[
2]
self.ep_value['surrogate_loss'], self.ep_value['kl_after'] = vars[
3], vars[4]
self.ep_value['entropy_after'], self.ep_value['value_loss'] = vars[
5], vars[6]
self.relative_time = (time.time() - self.start_time) / 60
# write_log with tensorboard: ['reward', 'kl_div', 'entropy', 'surrogate_loss', 'value_loss']
self.write_log([
self.ep_value['episode_reward'], self.ep_value['kl_after'],
self.ep_value['entropy_after'], self.ep_value['surrogate_loss'],
self.ep_value['value_loss']
], self.iteration)
# print iteration information:
episode_print(self.iteration, self.total_steps, self.relative_time,
self.ep_value)
self.history_reward.append(self.ep_value["episode_reward"])
def rollout_an_episode(self, env):
global graph
with graph.as_default():
path = self.agent.run_an_episode(env)
self.lock.acquire()
self.ep_num += 1
self.rollout_count += 1
self.rollout_paths.append(path)
self.total_steps += len(path["actions"])
self.lock.release()
env.rel()
def rollout_if_available(self):
while True:
remote_env = self.farmer.acq_env() # call for a remote_env
if remote_env is False: # no free environment
# time.sleep(0.1)
pass
else:
t = th.Thread(target=self.rollout_an_episode,
args=(remote_env, ),
daemon=True)
t.start()
break
def rollout(self):
while self.total_steps < self.agent.args.n_steps:
if self.num_rollouts == 0:
raise RuntimeError('wrong, div 0!!!')
self.iteration += 1
for i in range(self.num_rollouts):
self.rollout_if_available()
while self.rollout_count != self.num_rollouts:
pass
if (self.iteration + 1) % 10 == 0:
self.agent.saveModel(0)
self.average_len_of_episode = sum(
[len(path["rewards"])
for path in self.rollout_paths]) / self.num_rollouts
if self.average_len_of_episode == 0:
raise RuntimeError('wrong, div 0!!!')
self.fit()
self.rollout_count = 0
self.rollout_paths = []
self.num_rollouts = int(self.agent.args.timesteps_per_batch /
self.average_len_of_episode)
self.agent.saveModel(0)
return self.history_reward
class on_policy_parallel_rollouts():
def __init__(self, agent, env=None):
self.lock = th.Lock()
self.agent = agent
# one and only
self.farmer = farmer_class(self.agent.para_list)
self.ep_num = 0
self.total_steps = 0
self.history_reward = []
self.ep_value = {}
self.relative_time = 0
self.average_len_of_episode = self.agent.para_list["max_pathlength"]
self.num_rollouts = int(self.agent.para_list["timesteps_per_batch"] /
self.average_len_of_episode)
self.rollout_count = 0
self.rollout_paths = []
self.iteration = 0
self.log_scalar_name_list = [
'mean_reward', 'kl_div', 'entropy', 'surrogate_loss', 'value_loss'
]
self.log_scalar_type_list = [
tf.float32, tf.float32, tf.float32, tf.float32, tf.float32
]
self.logger = Logger(self.agent.session,
self.agent.para_list["log_path"] + 'train',
self.log_scalar_name_list,
self.log_scalar_type_list)
self.write_log = self.logger.create_scalar_log_method()
self.start_time = time.time()
def refarm(self): # most time no use
del self.farmer
self.farmer = farmer_class()
def fit(self):
# calculate real leaning rate with a linear schedule
learning_rate = learning_rate_schedule(self.agent.para_list,
self.total_steps)
# vars: update_old_policy_time, train_time, fit_time, surrogate_loss, kl_after, entropy_after, value_loss
vars = self.agent.fit(self.rollout_paths, learning_rate)
# update print info:
self.ep_value["learning_rate"] = (learning_rate[0] +
learning_rate[1]) / 2
self.ep_value["episode_nums"] = self.num_rollouts
self.ep_value["episode_steps"] = sum(
[len(path["rewards"]) for path in self.rollout_paths])
self.ep_value["episode_time"] = sum(
[path["ep_time"]
for path in self.rollout_paths]) / self.num_rollouts
self.ep_value["episode_reward"] = sum(
[path["rewards"].sum()
for path in self.rollout_paths]) / self.num_rollouts
self.ep_value["average_steps"] = int(self.average_len_of_episode)
self.ep_value["step_time"] = sum(
[path["step_time"]
for path in self.rollout_paths]) / self.num_rollouts
# copy fit info to the ep_value
self.ep_value['update_time'], self.ep_value[
'train_time'], self.ep_value['iter_time'] = vars[0], vars[1], vars[
2]
self.ep_value['surrogate_loss'], self.ep_value['kl_after'] = vars[
3], vars[4]
self.ep_value['entropy_after'], self.ep_value['value_loss'] = vars[
5], vars[6]
self.relative_time = (time.time() - self.start_time) / 60
# write_log with tensorboard: ['reward', 'kl_div', 'entropy', 'surrogate_loss', 'value_loss']
self.write_log([
self.ep_value['episode_reward'], self.ep_value['kl_after'],
self.ep_value['entropy_after'], self.ep_value['surrogate_loss'],
self.ep_value['value_loss']
], self.iteration)
# print iteration information:
on_policy_iteration_print(self.iteration, self.total_steps,
self.relative_time, self.ep_value)
self.history_reward.append(self.ep_value["episode_reward"])
def rollout_an_episode(self, env):
global graph
with graph.as_default():
path = rollout_an_episode(self.agent, env, self.lock)
self.lock.acquire()
self.ep_num += 1
self.rollout_count += 1
self.rollout_paths.append(path)
self.total_steps += len(path["actions"])
self.lock.release()
env.rel()
def rollout_if_available(self):
while True:
remote_env = self.farmer.acq_env() # call for a remote_env
if remote_env is False: # no free environment
# time.sleep(0.1)
pass
else:
t = th.Thread(target=self.rollout_an_episode,
args=(remote_env, ),
daemon=True)
t.start()
break
def rollout(self):
while self.total_steps < self.agent.para_list["n_steps"]:
if self.num_rollouts == 0:
raise RuntimeError('wrong, div 0!!!')
self.iteration += 1
for i in range(self.num_rollouts):
self.rollout_if_available()
while self.rollout_count != self.num_rollouts:
pass
if (self.iteration + 1) % 10 == 0:
self.agent.saveModel(0)
self.average_len_of_episode = sum(
[len(path["rewards"])
for path in self.rollout_paths]) / self.num_rollouts
if self.average_len_of_episode == 0:
raise RuntimeError('wrong, div 0!!!')
self.fit()
self.rollout_count = 0
self.rollout_paths = []
self.num_rollouts = int(
self.agent.para_list["timesteps_per_batch"] /
self.average_len_of_episode)
self.agent.saveModel(0)
return self.history_reward