def controller_test_proc(controller, vae, mdrnn):
step_log('4-3. controller_test_proc START!!')
# define current best and load parameters
if not os.path.exists(ctrl_dir):
os.mkdir(ctrl_dir)
ctrl_file = os.path.join(ctrl_dir, 'best.tar')
print("Attempting to load previous best...")
if os.path.exists(ctrl_file):
# state = torch.load(ctrl_file, map_location={'cuda:0': 'cpu'})
state = torch.load(ctrl_file)
controller.load_state_dict(state['state_dict'])
print("Controller Test Rollout START!!")
with torch.no_grad():
r_gen = RolloutGenerator(vae, mdrnn, controller, device,
rollout_time_limit)
r_gen.rollout(flatten_parameters(controller.parameters()), render=True)
def train_explorer(logdir,
epochs=10,
n_samples=4,
pop_size=4,
display=True,
max_workers=10):
results = {}
results['best'] = []
# multiprocessing variables
num_workers = min(max_workers, n_samples * pop_size)
time_limit = 1000
# create tmp dir if non existent and clean it if existent
tmp_dir = join(logdir, 'tmp_exp')
if not exists(tmp_dir):
mkdir(tmp_dir)
else:
for fname in listdir(tmp_dir):
unlink(join(tmp_dir, fname))
# create exp dir if non exitent
explore_dir = join(logdir, 'explore')
if not exists(explore_dir):
mkdir(explore_dir)
################################################################################
# Thread routines #
################################################################################
def slave_routine(p_queue, r_queue, e_queue, p_index):
""" Thread routine.
Threads interact with p_queue, the parameters queue, r_queue, the result
queue and e_queue the end queue. They pull parameters from p_queue, execute
the corresponding rollout, then place the result in r_queue.
Each parameter has its own unique id. Parameters are pulled as tuples
(s_id, params) and results are pushed as (s_id, result). The same
parameter can appear multiple times in p_queue, displaying the same id
each time.
As soon as e_queue is non empty, the thread terminate.
When multiple gpus are involved, the assigned gpu is determined by the
process index p_index (gpu = p_index % n_gpus).
:args p_queue: queue containing couples (s_id, parameters) to evaluate
:args r_queue: where to place results (s_id, results)
:args e_queue: as soon as not empty, terminate
:args p_index: the process index
"""
# init routine
gpu = p_index % torch.cuda.device_count()
device = torch.device(
'cuda:{}'.format(gpu) if torch.cuda.is_available() else 'cpu')
# redirect streams
sys.stdout = open(join(tmp_dir, str(getpid()) + '.out'), 'a')
sys.stderr = open(join(tmp_dir, str(getpid()) + '.err'), 'a')
# with torch.no_grad():
# r_gen = RolloutGenerator(logdir, device, time_limit)
# while e_queue.empty():
# if p_queue.empty():
# sleep(.1)
# else:
# s_id, params = p_queue.get()
# r_queue.put((s_id, r_gen.rollout(params)))
with torch.no_grad():
r_gen = RolloutGenerator(logdir, device, time_limit)
while e_queue.empty():
if p_queue.empty():
sleep(.1)
else:
s_id, params = p_queue.get()
r_queue.put((s_id, r_gen.rollout(params)))
################################################################################
# Define queues and start workers #
################################################################################
p_queue = Queue()
r_queue = Queue()
e_queue = Queue()
for p_index in range(num_workers):
Process(target=slave_routine,
args=(p_queue, r_queue, e_queue, p_index)).start()
################################################################################
# Evaluation #
################################################################################
def evaluate(solutions, results, rollouts=100):
""" Give current controller evaluation.
Evaluation is minus the cumulated reward averaged over rollout runs.
:args solutions: CMA set of solutions
:args results: corresponding results
:args rollouts: number of rollouts
:returns: minus averaged cumulated reward
"""
index_min = np.argmin(results)
best_guess = solutions[index_min]
restimates = []
for s_id in range(rollouts):
p_queue.put((s_id, best_guess))
print("Evaluating...")
for _ in tqdm(range(rollouts)):
while r_queue.empty():
sleep(.1)
restimates.append(r_queue.get()[1])
return best_guess, np.mean(restimates), np.std(restimates)
################################################################################
# Launch CMA #
################################################################################
controller = Controller(LSIZE, RSIZE, ASIZE) # dummy instance
# define current best and load parameters
cur_best = None
ctrl_file = join(explore_dir, 'best.tar')
print("Attempting to load previous best...")
if exists(ctrl_file):
state = torch.load(ctrl_file, map_location={'cuda:0': 'cpu'})
cur_best = -state['reward']
controller.load_state_dict(state['state_dict'])
print("Previous best was {}...".format(-cur_best))
parameters = controller.parameters()
es = cma.CMAEvolutionStrategy(flatten_parameters(parameters), 0.1,
{'popsize': pop_size})
epoch = 0
log_step = 3
while not es.stop():
if cur_best is not None and -cur_best > target_return:
print("Already better than target, breaking...")
break
r_list = [0] * pop_size # result list
solutions = es.ask()
# push parameters to queue
for s_id, s in enumerate(solutions):
for _ in range(n_samples):
p_queue.put((s_id, s))
# retrieve results
if display:
pbar = tqdm(total=pop_size * n_samples)
for _ in range(pop_size * n_samples):
while r_queue.empty():
sleep(.1)
r_s_id, r = r_queue.get()
r_list[r_s_id] += r / n_samples
if display:
pbar.update(1)
if display:
pbar.close()
es.tell(solutions, r_list)
es.disp()
# evaluation and saving
if epoch % log_step == log_step - 1:
best_params, best, std_best = evaluate(solutions, r_list)
# log the best
results['best'].append(best)
print("Current evaluation: {}".format(best))
if not cur_best or cur_best > best:
cur_best = best
print("Saving new best with value {}+-{}...".format(
-cur_best, std_best))
load_parameters(best_params, controller)
torch.save(
{
'epoch': epoch,
'reward': -cur_best,
'state_dict': controller.state_dict()
}, join(explore_dir, 'best.tar'))
if -best > target_return:
print(
"Terminating controller training with value {}...".format(
best))
break
epoch += 1
es.result_pretty()
e_queue.put('EOP')
return results
controller = Controller(LSIZE, RSIZE, ASIZE) # dummy instance
# define current best and load parameters
cur_best = 0
ctrl_file = join(ctrl_dir, 'best.tar')
print("Attempting to load previous best...")
if exists(ctrl_file):
state = torch.load(ctrl_file, map_location={'cuda:0': 'cpu'})
cur_best = -state['reward']
controller.load_state_dict(state['state_dict'])
print("Previous best was {}...".format(-cur_best))
else:
print('init controller')
parameters = controller.parameters()
es = cma.CMAEvolutionStrategy(flatten_parameters(parameters), 0.1,
{'popsize': pop_size})
epoch = 0
log_step = 3
print('start CMA-ES')
count_g = 0
while not es.stop():
if cur_best is not None and -cur_best > args.target_return:
print("Already better than target, breaking...")
break
print('training_es', count_g)
r_list = [0] * pop_size # result list
solutions = es.ask()
# push parameters to queue
# Launch CMA #
################################################################################
controller = Controller(LSIZE, RSIZE, ASIZE) # dummy instance
# define current best and load parameters
cur_best = None
ctrl_file = join(ctrl_dir, 'best.tar')
print("Attempting to load previous best...")
if exists(ctrl_file):
state = torch.load(ctrl_file, map_location={'cuda:0': 'cpu'})
cur_best = - state['reward']
controller.load_state_dict(state['state_dict'])
print("Previous best was {}...".format(-cur_best))
parameters = controller.parameters()
es = cma.CMAEvolutionStrategy(flatten_parameters(parameters), 0.1,
{'popsize': pop_size})
epoch = 0
log_step = 3
while not es.stop():
if cur_best is not None and - cur_best > args.target_return:
print("Already better than target, breaking...")
break
r_list = [0] * pop_size # result list
solutions = es.ask()
# push parameters to queue
for s_id, s in enumerate(solutions):
for _ in range(n_samples):
def run(args):
p_queue = Queue()
r_queue = Queue()
e_queue = Queue()
latent = 32
mixture = 256
size = latent + mixture
controller = Controller(size, 3)
for i in range(args.max_workers):
Process(target=slave_routine,
args=(p_queue, r_queue, e_queue, i, args.logdir)).start()
cur_best = None
savefile = args.logdir/'best.tar'
if savefile.exists():
print(f'Loading from {savefile}')
state = torch.load(savefile.as_posix(), map_location={'cuda:0': 'cpu'})
cur_best = -state['reward']
controller.load_state_dict(state['state_dict'])
parameters = controller.parameters()
sigma = 0.1
es = cma.CMAEvolutionStrategy(flatten_parameters(parameters), sigma,
{'popsize': args.pop_size})
epoch = 0
while not es.stop():
if cur_best is not None and -cur_best > args.target_return:
print('Already better than target, breaking...')
break
r_list = [0] * args.pop_size # result list
solutions = es.ask()
# push parameters to queue
for s_id, s in enumerate(solutions):
for _ in range(args.n_samples):
p_queue.put((s_id, s))
# Retrieve results
if args.display:
pbar = tqdm(total=args.pop_size * args.n_samples)
for _ in range(args.pop_size * args.n_samples):
while r_queue.empty():
sleep(.1)
r_s_id, r = r_queue.get()
r_list[r_s_id] += r / args.n_samples
if args.display:
pbar.update(1)
if args.display:
pbar.close()
es.tell(solutions, r_list)
es.disp()
# CMA-ES seeks to minimize, so we want to multiply the reward we
# get in a rollout by -1.
best_params, best, std_best = evaluate(solutions, r_list, p_queue,
r_queue)
if (not cur_best) or (cur_best > best):
cur_best = best
print(f'Saving new best with value {-cur_best}+{-std_best}')
load_parameters(best_params, controller)
torch.save({'epoch': epoch,
'reward': -cur_best,
'state_dict': controller.state_dict()},
savefile)
# Save after every epoch
torch.save(controller.state_dict(), f'{controller_pt}')
if -best > args.target_return:
print(f'Terminating controller training with value {best}...')
break
epoch += 1
es.result_pretty()
e_queue.put('EOP')
def controller_train_proc(ctrl_dir,
controller,
vae,
mdrnn,
target_return=950,
skip_train=False,
display=True):
step_log('4-2. controller_train_proc START!!')
# define current best and load parameters
cur_best = None
if not os.path.exists(ctrl_dir):
os.mkdir(ctrl_dir)
ctrl_file = os.path.join(ctrl_dir, 'best.tar')
p_queue = Queue()
r_queue = Queue()
#e_queue = Queue() # pipaek : not necessary if not multiprocessing
print("Attempting to load previous best...")
if os.path.exists(ctrl_file):
#state = torch.load(ctrl_file, map_location={'cuda:0': 'cpu'})
state = torch.load(ctrl_file)
cur_best = -state['reward']
controller.load_state_dict(state['state_dict'])
print("Previous best was {}...".format(-cur_best))
if skip_train:
return # pipaek : 트레이닝을 통한 모델 개선을 skip하고 싶을 때..
def evaluate(solutions,
results,
rollouts=100): # pipaek : rollout 100 -> 10 , originally 100
""" Give current controller evaluation.
Evaluation is minus the cumulated reward averaged over rollout runs.
:args solutions: CMA set of solutions
:args results: corresponding results
:args rollouts: number of rollouts
:returns: minus averaged cumulated reward
"""
index_min = np.argmin(results)
best_guess = solutions[index_min]
restimates = []
for s_id in range(rollouts):
print('p_queue.put(), s_id=%d' % s_id)
p_queue.put((s_id, best_guess))
print('>>>rollout_routine!!')
rollout_routine() # pipaek : 여기서도 p_queue.put 하자마자 바로 처리..
print(">>>Evaluating...")
for _ in tqdm(range(rollouts)):
#while r_queue.empty():
# sleep(.1) # pipaek : multi-process가 아니므로
if not r_queue.empty(
): # pipaek : 20180718 r_queue.get()에서 stuck되어 있는 것을 방지하기 위해 체크!!
#print('r_queue.get()')
#restimates.append(r_queue.get()[1])
r_s_id, r = r_queue.get()
print(
'in evaluate r_queue.get() r_s_id=%d, r_queue remain=%d' %
(r_s_id, r_queue.qsize()))
restimates.append(r)
else:
print('r_queue.empty() -> break!!')
break
return best_guess, np.mean(restimates), np.std(restimates)
def rollout_routine():
""" Thread routine.
Threads interact with p_queue, the parameters queue, r_queue, the result
queue and e_queue the end queue. They pull parameters from p_queue, execute
the corresponding rollout, then place the result in r_queue.
Each parameter has its own unique id. Parameters are pulled as tuples
(s_id, params) and results are pushed as (s_id, result). The same
parameter can appear multiple times in p_queue, displaying the same id
each time.
As soon as e_queue is non empty, the thread terminate.
When multiple gpus are involved, the assigned gpu is determined by the
process index p_index (gpu = p_index % n_gpus).
:args p_queue: queue containing couples (s_id, parameters) to evaluate
:args r_queue: where to place results (s_id, results)
:args e_queue: as soon as not empty, terminate
:args p_index: the process index
"""
# init routine
#gpu = p_index % torch.cuda.device_count()
#device = torch.device('cuda:{}'.format(gpu) if torch.cuda.is_available() else 'cpu')
# redirect streams
#if not os.path.exists(tmp_dir):
# os.mkdir(tmp_dir)
#sys.stdout = open(os.path.join(tmp_dir, 'rollout.out'), 'a')
#sys.stderr = open(os.path.join(tmp_dir, 'rollout.err'), 'a')
with torch.no_grad():
r_gen = RolloutGenerator(vae, mdrnn, controller, device,
rollout_time_limit)
while not p_queue.empty():
print('in rollout_routine, p_queue.get()')
s_id, params = p_queue.get()
print('r_queue.put() sid=%d' % s_id)
r_queue.put((s_id, r_gen.rollout(params)))
print('r_gen.rollout OK, r_queue.put()')
#r_queue.qsize()
parameters = controller.parameters()
es = cma.CMAEvolutionStrategy(flatten_parameters(parameters), 0.1,
{'popsize': C_POP_SIZE})
print("CMAEvolutionStrategy start OK!!")
epoch = 0
log_step = 3
while not es.stop():
print("--------------------------------------")
print("CURRENT EPOCH = %d" % epoch)
if cur_best is not None and -cur_best > target_return:
print("Already better than target, breaking...")
break
r_list = [0] * C_POP_SIZE # result list
solutions = es.ask()
print("CMAEvolutionStrategy-ask")
# push parameters to queue
for s_id, s in enumerate(
solutions): # pipaek : 이 for가 C_POP_SIZE 만큼 반복된다.
#for _ in range(C_POP_SIZE * C_N_SAMPLES):
for _ in range(C_N_SAMPLES):
print('in controller_train_proc p_queue.put() s_id : %d' %
s_id)
p_queue.put((s_id, s))
#print("p_queue.put %d" % s_id)
rollout_routine(
) # pipaek : p_queue.put 하자마자 바로 get해서 rollout하고 나서 r_queue에 결과 입력.
print("rollout_routine OK, r_queue size=%d" % r_queue.qsize())
# retrieve results
if display:
pbar = tqdm(total=C_POP_SIZE * C_N_SAMPLES)
#for idx in range(C_POP_SIZE * C_N_SAMPLES):
while not r_queue.empty(
): # pipaek : 20180718 여기서 r_queue.get을 못해서 영원히 걸려있는 상태를 방지하기 위해 for문을 while문으로 바꾼다.
#while r_queue.empty():
# sleep(.1)
try:
r_s_id, r = r_queue.get()
print(
'in controller_train_proc r_queue.get() r_s_id=%d, r_queue remain=%d'
% (r_s_id, r_queue.qsize()))
r_list[r_s_id] += r / C_N_SAMPLES
if display:
pbar.update(1)
except IndexError as err:
print('IndexError during r_queue.get()')
print('cur r_list size:%d, index:%d' % (len(r_list), r_s_id))
if display:
pbar.close()
es.tell(solutions,
r_list) # pipaek : solution array에다가 r_list 결과를 업데이트..
es.disp()
# evaluation and saving
if epoch % log_step == log_step - 1:
print(">>>> TRYING EVALUATION, CURRENT EPOCH = %d" % epoch)
best_params, best, std_best = evaluate(
solutions, r_list, rollouts=100
) # pipaek : evaluate을 위해서 rollout은 10번만 하자.. originally 100
print("Current evaluation: {}".format(best))
if not cur_best or cur_best > best:
cur_best = best
print("Saving new best with value {}+-{}...".format(
-cur_best, std_best))
load_parameters(best_params, controller)
torch.save(
{
'epoch': epoch,
'reward': -cur_best,
'state_dict': controller.state_dict()
}, os.path.join(ctrl_dir, 'best.tar'))
if -best > target_return:
print(
"Terminating controller training with value {}...".format(
best))
break
epoch += 1
print("es.stop!!")
es.result_pretty()
controller = Controller(LSIZE, RSIZE, ASIZE) # dummy instance
# define current best and load parameters
cur_best = None
ctrl_file = join(prev_ctrl_dir, "best.tar")
print("Attempting to load previous best...")
if exists(ctrl_file):
state = torch.load(ctrl_file, map_location={"cuda:0": "cpu"})
# cur_best = -state["reward"]
print("Loading Controller from {}".format(ctrl_file))
controller.load_state_dict(state["state_dict"])
# print("Previous best was {}...".format(-cur_best))
parameters = controller.parameters()
es = cma.CMAEvolutionStrategy(
flatten_parameters(parameters), 0.1, {"popsize": pop_size}
)
epoch = 0
log_step = 1
while not es.stop():
if epoch == 5:
print("Already better than target, breaking...")
break
r_list = [0] * pop_size # result list
solutions = es.ask()
# push parameters to queue
for s_id, s in enumerate(solutions):
for _ in range(n_samples):
