def mp__update_params(args, q_i_buf, q_o_buf, q_i_eva, q_o_eva): # update network parameters using replay buffer
class_agent = args.rl_agent
max_memo = args.max_memo
net_dim = args.net_dim
max_step = args.max_step
max_total_step = args.max_total_step
batch_size = args.batch_size
repeat_times = args.repeat_times
del args
state_dim, action_dim = q_o_buf.get() # q_o_buf 1.
agent = class_agent(state_dim, action_dim, net_dim)
from copy import deepcopy
act_cpu = deepcopy(agent.act).to(torch.device("cpu"))
act_cpu.eval()
[setattr(param, 'requires_grad', False) for param in act_cpu.parameters()]
q_i_buf.put(act_cpu) # q_i_buf 1.
# q_i_buf.put(act_cpu) # q_i_buf 2. # warning
q_i_eva.put(act_cpu) # q_i_eva 1.
buffer = BufferArrayGPU(max_memo, state_dim, action_dim) # experiment replay buffer
'''initial_exploration'''
buffer_array, reward_list, step_list = q_o_buf.get() # q_o_buf 2.
reward_avg = np.average(reward_list)
step_sum = sum(step_list)
buffer.extend_memo(buffer_array)
q_i_eva.put((act_cpu, reward_avg, step_sum, 0, 0)) # q_i_eva 1.
total_step = step_sum
is_training = True
while is_training:
buffer_array, reward_list, step_list = q_o_buf.get() # q_o_buf n.
reward_avg = np.average(reward_list)
step_sum = sum(step_list)
total_step += step_sum
buffer.extend_memo(buffer_array)
buffer.init_before_sample()
loss_a_avg, loss_c_avg = agent.update_parameters(buffer, max_step, batch_size, repeat_times)
act_cpu.load_state_dict(agent.act.state_dict())
q_i_buf.put(act_cpu) # q_i_buf n.
q_i_eva.put((act_cpu, reward_avg, step_sum, loss_a_avg, loss_c_avg)) # q_i_eva n.
if q_o_eva.qsize() > 0:
is_solved = q_o_eva.get() # q_o_eva n.
if is_solved:
is_training = False
if total_step > max_total_step:
is_training = False
q_i_buf.put('stop')
q_i_eva.put('stop')
while q_i_buf.qsize() > 0 or q_i_eva.qsize() > 0:
time.sleep(1)
time.sleep(4)
print('; quit: params')
def mp__update_params(args, q_i_buf, q_o_buf, q_i_eva, q_o_eva): # update network parameters using replay buffer
class_agent = args.class_agent
max_memo = args.max_memo
net_dim = args.net_dim
max_epoch = args.max_epoch
max_step = args.max_step
batch_size = args.batch_size
repeat_times = args.repeat_times
args.init_for_training()
del args
state_dim, action_dim = q_o_buf.get() # q_o_buf 1.
agent = class_agent(state_dim, action_dim, net_dim)
from copy import deepcopy
act_cpu = deepcopy(agent.act).to(torch.device("cpu"))
act_cpu.eval()
[setattr(param, 'requires_grad', False) for param in act_cpu.parameters()]
q_i_buf.put(act_cpu) # q_i_buf 1.
# q_i_buf.put(act_cpu) # q_i_buf 2. # warning
q_i_eva.put(act_cpu) # q_i_eva 1.
buffer = BufferArrayGPU(max_memo, state_dim, action_dim) # experiment replay buffer
'''initial_exploration'''
buffer_array, reward_list, step_list = q_o_buf.get() # q_o_buf 2.
reward_avg = np.average(reward_list)
step_sum = sum(step_list)
buffer.extend_memo(buffer_array)
q_i_eva.put((act_cpu, reward_avg, step_sum, 0, 0)) # q_i_eva 1.
for epoch in range(max_epoch): # epoch is episode
buffer_array, reward_list, step_list = q_o_buf.get() # q_o_buf n.
reward_avg = np.average(reward_list)
step_sum = sum(step_list)
buffer.extend_memo(buffer_array)
buffer.init_before_sample()
loss_a_avg, loss_c_avg = agent.update_parameters(buffer, max_step, batch_size, repeat_times)
act_cpu.load_state_dict(agent.act.state_dict())
q_i_buf.put(act_cpu) # q_i_buf n.
q_i_eva.put((act_cpu, reward_avg, step_sum, loss_a_avg, loss_c_avg)) # q_i_eva n.
if q_o_eva.qsize() > 0:
is_solved = q_o_eva.get() # q_o_eva n.
if is_solved:
break
def mp__update_params(args, q_i_buf, q_o_buf, q_i_eva, q_o_eva): # update network parameters using replay buffer
class_agent = args.rl_agent
max_memo = args.max_memo
net_dim = args.net_dim
max_step = args.max_step
max_total_step = args.break_step
batch_size = args.batch_size
repeat_times = args.repeat_times
cwd = args.cwd
env_name = args.env_name
if_stop = args.if_break_early
del args
state_dim, action_dim = q_o_buf.get() # q_o_buf 1.
agent = class_agent(state_dim, action_dim, net_dim)
from copy import deepcopy
act_cpu = deepcopy(agent.act).to(torch.device("cpu"))
act_cpu.eval()
[setattr(param, 'requires_grad', False) for param in act_cpu.parameters()]
q_i_buf.put(act_cpu) # q_i_buf 1.
q_i_eva.put(act_cpu) # q_i_eva 1.
buffer = BufferArrayGPU(max_memo, state_dim, action_dim) # experiment replay buffer
'''initial_exploration'''
buffer_array, reward_list, step_list = q_o_buf.get() # q_o_buf 2.
reward_avg = np.average(reward_list)
step_sum = sum(step_list)
buffer.extend_memo(buffer_array)
'''pre training and hard update before training loop'''
buffer.init_before_sample()
agent.update_parameters(buffer, max_step, batch_size, repeat_times)
agent.act_target.load_state_dict(agent.act.state_dict())
q_i_eva.put((act_cpu, reward_avg, step_sum, 0, 0)) # q_i_eva 1.
total_step = step_sum
if_train = True
if_solve = False
while if_train:
buffer_array, reward_list, step_list = q_o_buf.get() # q_o_buf n.
reward_avg = np.average(reward_list)
step_sum = sum(step_list)
total_step += step_sum
buffer.extend_memo(buffer_array)
buffer.init_before_sample()
loss_a_avg, loss_c_avg = agent.update_parameters(buffer, max_step, batch_size, repeat_times)
act_cpu.load_state_dict(agent.act.state_dict())
q_i_buf.put(act_cpu) # q_i_buf n.
q_i_eva.put((act_cpu, reward_avg, step_sum, loss_a_avg, loss_c_avg)) # q_i_eva n.
if q_o_eva.qsize() > 0:
if_solve = q_o_eva.get() # q_o_eva n.
'''break loop rules'''
if_train = not ((if_stop and if_solve)
or total_step > max_total_step
or os.path.exists(f'{cwd}/stop'))
env, state_dim, action_dim, target_reward, if_discrete = build_gym_env(env_name, if_print=False)
buffer.print_state_norm(env.neg_state_avg, env.div_state_std)
q_i_buf.put('stop')
q_i_eva.put('stop')
while q_i_buf.qsize() > 0 or q_i_eva.qsize() > 0:
time.sleep(1)
time.sleep(4)