print('Init error')
# Create Data buffer
exp_data = DataBuffer(env, max_trajectory = num_iter_algo*10 +n_rnd, shaping_state_delta = shaping_state_delta)
# during first n_rnd trials, apply randomized controls
for i in range(n_rnd):
exp_data.push(rollout(env, randpol, max_steps=T))
#cost_mean ,cost_std = test_episodic_cost(env, policy, N=50, T=T, render=False)
for i in range( num_iter_algo):
log.infov('-----------------DeepPILCO Iteration # {}-----------------'.format(i+1))
# Train dynamics
train_dynamics_model_pilco(dynamics, dynamics_optimizer, exp_data, epochs=num_itr_dyn, batch_size=dyn_batch_size, plot_train=None, pre_process=pre_process)
# Update policy
log.infov('Policy optimization...' )
policy.update_dataset_statistics(exp_data)
for j in range(num_iter_policy):
_, list_costs, list_moments = learn_policy_pilco(env, dynamics, policy, policy_optimizer, K=K, T= 1000, gamma=0.99,
moment_matching=True, grad_norm = grad_clip, pre_prcess=True , shaping_state_delta= shaping_state_delta)
# Loggings
if (j + 1) % log_interval_policy == 1 or (j + 1) == args.num_iter_policy:
loss_mean = torch.sum( torch.cat(list_costs)) .data.cpu().numpy()[0]
def train_dynamics_model_pilco2(dynamics, dynamics_optimizer, trainset, epochs=1, batch_size=1, eval_fn=None,
logger=None, **kwargs):
# Create dynamics and its optimizer
dynamics.set_sampling(sampling=False)
log.infov('Dynamics training...')
# Loss
#criterion = nn.MSELoss() # MSE/SmoothL1
dynamics.update_dataset_statistics(trainset)
batch_size = trainset.data.shape[0] if trainset.data.shape[0] < batch_size else batch_size
# Create Dataloader
trainloader = Data.DataLoader(trainset, batch_size=batch_size, shuffle=True, drop_last=True)
(_, _), (x_test, y_test) = load_data()
(_, _), (x_test2, y_test2) = load_data(
dir_name='/home/drl/PycharmProjects/DeployedProjects/deepPILCO/MB/data/log-test1.csv', data_num=1000)
log.infov('Num of rollout: {} Data set size: {}'.format(len(trainset.buffer), trainset.data.shape[0]))
#dynamics.set_sampling(sampling= False)
dynamics.train()
list_train_loss = []
for epoch in range(epochs): # Loop over dataset multiple times
running_train_losses = []
start_time = time.time()
for i, data in enumerate(trainloader): # Loop over batches of data
# Get input batch
X, Y = data
# Loss
loss = dynamics.get_loss( X, Y, pre_prcess = kwargs['pre_process'])
# Backward pass
loss.backward()
# Update params
dynamics_optimizer.step()
# Accumulate running losses
running_train_losses.append(loss.data[0]) # Take out value from 1D Tensor
# Record the mean of training and validation losses in the batch
batch_train_loss = np.mean(running_train_losses)
list_train_loss.append(batch_train_loss)
time_duration = time.time() - start_time
# Logging: Only first, middle and last
if epoch % LOG_EVERY_N_EPOCH == 0:
if dynamics.env.spec.id == 'HalfCheetah-v2':
eval_mse = plot_train(x_test, y_test, dyn_model=dynamics, pre_process=kwargs['pre_process'], plot=False)
eval_mse2 = plot_train(x_test2, y_test2, dyn_model=dynamics, pre_process=kwargs['pre_process'],
plot=False)
# log.info('[Epoch # {:3d} ({:.1f} s)] Train loss: {:.8f} Eval loss1: {:.8f} Eval loss2: {:.8f}'.format(epoch + 1, time_duration, batch_train_loss, eval_mse, eval_mse2))
else:
eval_mse = 0
eval_mse2 = 0
if logger is not None:
logger.log({'epoch': epoch,
'time_duration': time_duration,
'Train loss': batch_train_loss,
'Eval loss': eval_mse,
'Eval loss_export': eval_mse2,
})
logger.write(display=False)
if epoch == 0 or epoch == epochs // 2 or epoch == epochs - 1 or epoch % 5 == 0:
log.info(
'[Epoch # {:3d} ({:.1f} s)] Train loss: {:.8f} Eval loss1: {:.8f} Eval loss2: {:.8f}'.format(epoch + 1,
time_duration,
batch_train_loss,
eval_mse,
eval_mse2))
if epoch % PLOT_EVERY_N_EPOCH == 0:
if kwargs['plot_train'] is not None:
if callable(kwargs['plot_train']):
if epoch == 0:
plt.ion()
kwargs['plot_train'](dynamics)
if logger is not None:
logger.close()
log.info('Finished training dynamics model. ')
return np.array(list_train_loss)
cost_fn=cost_fn,
num_simulated_paths=simulated_paths,
action_noise=action_noise,
N_SAMPLES=10,
)
# Create Data buffer
exp_data = DataBuffer(
env, max_trajectory=n_rnd +
3 * N_MPC) # num_iter_algo +n_rnd n_rnd + n_iter_algo* N_MPC
# during first n_rnd trials, apply randomized controls
for i in range(n_rnd):
exp_data.push(rollout(env, randpol, max_steps=max_timestep))
log.infov(
'-----------------DeepPILCO Iteration # {}-----------------'.format(i + 1))
#Train dynamics
train_dynamics_model_pilco(dynamics,
dynamics_optimizer,
exp_data,
epochs=num_itr_dyn,
batch_size=dyn_batch_size,
plot_train=None,
pre_process=pre_process)
#dynamics.update_dataset_statistics(exp_data)
# Save model
save_dir = log_dir
utils.save_net_param(dynamics, save_dir, name='dyn_model0', mode='net')
# exp_logger = utils.Logger(log_dir, csvname='exp' )
# data = np.concatenate((exp_data.buffer[0], exp_data.buffer[1],exp_data.buffer[2],exp_data.buffer[3],exp_data.buffer[4]), axis=0)
def train_dynamics_model_pilco(dynamics, dynamics_optimizer, trainset, epochs=1, batch_size=1, eval_fn=None,logger=None ,**kwargs):
# Create dynamics and its optimizer
dynamics.set_sampling(sampling=False)
log.infov('Dynamics training...')
# Loss
criterion = nn.MSELoss() # MSE/SmoothL1
dynamics.update_dataset_statistics(trainset)
batch_size = trainset.data.shape[0] if trainset.data.shape[0] < batch_size else batch_size
# Create Dataloader
trainloader = Data.DataLoader(trainset, batch_size=batch_size, shuffle=True, drop_last=True)
log.infov('Num of rollout: {} Data set size: {}'.format(len(trainset.buffer), trainset.data.shape[0]))
dynamics.train()
list_train_loss = []
for epoch in range(epochs): # Loop over dataset multiple times
running_train_losses = []
start_time = time.time()
for i, data in enumerate(trainloader): # Loop over batches of data
# Get input batch
X, Y = data
# Wrap data tensors as Variable and send to GPU
X = Variable(X).cuda()
Y = Variable(Y).cuda()
# Zero out the parameter gradients
dynamics_optimizer.zero_grad()
# Forward pass
outputs = dynamics.predict_Y(X, delta_target=True, pre_prcess=kwargs[
'pre_process']) # delta_target, return state difference for training
# Loss
loss = criterion(outputs, Y)
M = Y.shape[0]
N = X.shape[0]
# loss = gaussian_log_likelihood(Y,outputs)
reg = 0 # dropout_gp_kl(dynamics , input_lengthscale=1.0, hidden_lengthscale=1.0)
# loss = -loss/M + reg/N
# Backward pass
loss.backward()
# Update params
dynamics_optimizer.step()
# Accumulate running losses
running_train_losses.append(loss.data[0]) # Take out value from 1D Tensor
# Record the mean of training and validation losses in the batch
batch_train_loss = np.mean(running_train_losses)
list_train_loss.append(batch_train_loss)
time_duration = time.time() - start_time
# Logging: Only first, middle and last
if epoch % LOG_EVERY_N_EPOCH == 0:
if logger is not None:
logger.log({'epoch': epoch,
'time_duration':time_duration,
'Train loss': batch_train_loss,
})
logger.write(display=False)
if epoch == 0 or epoch == epochs // 2 or epoch == epochs - 1:
log.info(
'[Epoch # {:3d} ({:.1f} s)] Train loss: {:.8f} '.format(epoch + 1,
time_duration,
batch_train_loss))
if logger is not None:
logger.close()
log.info('Finished training dynamics model. ')
return np.array(list_train_loss)
activation=net_activation).cuda()
dynamics_optimizer = torch.optim.Adam(dynamics.parameters(),
lr=lr_dynamics,
weight_decay=dyn_reg2)
# Create random policy
randpol = controller.RandomPolicy(env)
# Create Data buffer
exp_data = DataBuffer(env, max_trajectory=100)
# during first n_rnd trials, apply randomized controls
for i in range(n_rnd):
exp_data.push(rollout(env, randpol, max_steps=T, render=False))
log.infov('-----------------DeepPILCO Iteration # {}-----------------')
# Train dynamics
train_dynamics_model_pilco(dynamics,
dynamics_optimizer,
exp_data,
epochs=num_itr_dyn,
batch_size=dyn_batch_size,
plot_train=None,
pre_process=pre_process,
logger=logger) #plot_train_ion
# Save model
save_dir = log_dir
utils.save_net_param(dynamics, save_dir, name='dyn_model', mode='net')