def solve(self):
""" Solve the problem with classical iLQR
"""
# Initialize the trajectory, F_matrix, objective_function_value_last, C_matrix and c_vector
self._trajectory = self._dynamic_model.eval_traj(
) # init feasible trajectory
C_matrix = self._obj_fun.eval_hessian_obj_fun(self._trajectory)
c_vector = self._obj_fun.eval_grad_obj_fun(self._trajectory)
F_matrix = self._dynamic_model.eval_grad_dynamic_model(
self._trajectory)
logger.info("[+ +] Initial Obj.Val.: %.5e" %
(self._obj_fun.eval_obj_fun(self._trajectory)))
# Start iteration
start_time = tm.time()
for i in range(self._max_iter):
if i == 1: # skip the compiling time
start_time = tm.time()
iter_start_time = tm.time()
K_matrix, k_vector = self.backward_pass(C_matrix, c_vector,
F_matrix)
backward_time = tm.time()
self._trajectory, C_matrix, c_vector, F_matrix, obj, isStop = self.forward_pass(
self._trajectory, K_matrix, k_vector)
forward_time = tm.time()
logger.info(
"[+ +] Iter.No.%3d BWTime:%.3e FWTime:%.3e Obj.Val.:%.5e"
% (i, backward_time - iter_start_time,
forward_time - backward_time, obj))
logger.save_to_json(trajectory=self._trajectory.tolist())
if isStop and self._is_check_stop:
break
end_time = tm.time()
logger.info("[+ +] Completed! All Time:%.5e" % (end_time - start_time))
def retrain(self,
dataset,
max_epoch=10000,
stopping_criterion=1e-3,
lr=1e-3):
logger.info("[+ +] Re-traning starts...")
loss_fun = nn.MSELoss()
# optimizer = optim.SGD(self.model.parameters(), lr=lr, momentum=0.9)
optimizer = optim.RAdam(self._model.parameters(),
lr=lr,
weight_decay=1e-4)
X_train, Y_train = self._process_data(dataset)
for epoch in range(max_epoch):
#### Training ###
self._model.train()
optimizer.zero_grad()
Y_prediction = self._model(X_train)
obj_train = loss_fun(Y_prediction, Y_train)
obj_train.backward()
optimizer.step()
if obj_train.item(
) < stopping_criterion or epoch % 100 == 0: # Check stopping criterion
logger.info("[+ +] Epoch: %5d Train Obj: %.5e" %
(epoch + 1, obj_train.item()))
if obj_train.item() < stopping_criterion:
logger.info("[+ +] Re-training finished!")
self._model.eval()
return
raise Exception("Maximum epoch in the retraining is reached!")
def solve(self):
""" Solve the problem with classical iLQR
"""
# Initialize the trajectory, F_matrix, objective_function_value_last, C_matrix and c_vector
self._trajectory = self._dynamic_model.eval_traj() # init feasible trajectory
C_matrix = self._obj_fun.eval_hessian_obj_fun(self._trajectory)
c_vector = self._obj_fun.eval_grad_obj_fun(self._trajectory)
F_matrix = self._dynamic_model.eval_grad_dynamic_model(
self._trajectory)
# Start iteration
logger.info("[+ +] Initial Obj.Val.: %.5e" %
(self._real_obj_fun.eval_obj_fun(self._trajectory)))
total_iter_no = -1
for j in self._t:
if j != self._t[0]: # update t parameter
add_param = self.get_obj_add_param()
add_param[:, -1] = j*np.ones((self.dynamic_model._T))
self.set_obj_add_param(add_param)
for i in range(self._max_iter):
total_iter_no += 1
if j == self._t[0] and i == 1: # skip the compiling time
start_time = tm.time()
iter_start_time = tm.time()
K_matrix, k_vector = self.backward_pass(
C_matrix, c_vector, F_matrix)
backward_time = tm.time()
self._trajectory, C_matrix, c_vector, F_matrix, _, isStop = self.forward_pass(
self._trajectory, K_matrix, k_vector)
forward_time = tm.time()
# do not care the value of log barrier
obj = self._real_obj_fun.eval_obj_fun(self._trajectory)
logger.info("[+ +] Total Iter.No.%3d Iter.No.%3d BWTime:%.3e FWTime:%.3e Obj.Val.:%.5e" % (
total_iter_no, i, backward_time-iter_start_time, forward_time-backward_time, obj))
logger.save_to_json(trajectory=self._trajectory.tolist())
if isStop and self._is_check_stop:
self.set_obj_fun_value(np.inf)
logger.info(
"[+ +] Complete One Inner Loop! The log barrier parameter t is %.5f" % (j) + " in this iteration!")
break
end_time = tm.time()
logger.info("[+ +] Completed! All Time:%.5e" % (end_time-start_time))
def print_params(self):
for key in self.kwargs:
logger.info("[+] " + key + " = " + str(self.kwargs[key]))
def solve(self):
trajectory = self._dynamic_model.eval_traj(
) # init feasible trajectory
init_obj = self._obj_fun.eval_obj_fun(trajectory)
logger.info("[+ +] Initial Obj.Val.: %.5e" % (init_obj))
self.set_obj_fun_value(init_obj)
new_data = []
result_obj_val = np.zeros(self._iLQR_max_iter)
result_iter_time = np.zeros(self._iLQR_max_iter)
re_train_stopping_criterion = self._training_stopping_criterion
for i in range(int(self._iLQR_max_iter)):
if i == 1: # skip the compiling time
start_time = tm.time()
iter_start_time = tm.time()
C_matrix = self._obj_fun.eval_hessian_obj_fun(trajectory)
c_vector = self._obj_fun.eval_grad_obj_fun(trajectory)
F_matrix = self._nn_dynamic_model.eval_grad_dynamic_model(
trajectory)
F_matrix = gaussian_filter1d(F_matrix,
sigma=self._gaussian_filter_sigma,
axis=0)
K_matrix, k_vector = self.backward_pass(C_matrix, c_vector,
F_matrix)
trajectory, C_matrix, c_vector, F_matrix, obj_val, isStop = self.forward_pass(
trajectory, K_matrix, k_vector)
if i < self._decay_rate_max_iters:
re_train_stopping_criterion = re_train_stopping_criterion * self._decay_rate
iter_end_time = tm.time()
iter_time = iter_end_time - iter_start_time
logger.info("[+ +] Iter.No.:%3d Iter.Time:%.3e Obj.Val.:%.5e" %
(
i,
iter_time,
obj_val,
))
result_obj_val[i] = obj_val
result_iter_time[i] = iter_time
if isStop:
if len(
new_data
) != 0: # Ensure the optimal trajectroy being in the dataset
trajectory_noisy = trajectory
else:
trajectory_noisy = self.dynamic_model.eval_traj(
action_traj=(
trajectory[:, self.dynamic_model.n:] +
np.random.normal(0, self._gaussian_noise_sigma, [
self.dynamic_model.T, self.dynamic_model.m, 1
])))
new_data += [trajectory_noisy]
data = np.concatenate(
new_data[-int(self._trial_no / 5):])[:, :, 0]
self._dataset_train.update_dataset(Data(state=data))
logger.save_to_json(trajectory=trajectory.tolist(),
trajectroy_noisy=trajectory_noisy.tolist())
self._nn_dynamic_model.retrain(
self._dataset_train,
max_epoch=100000,
stopping_criterion=re_train_stopping_criterion)
new_data = []
else:
new_data += [trajectory]
end_time = tm.time()
io.savemat(os.path.join(logger.logger_path, "_result.mat"), {
"obj_val": result_obj_val,
"iter_time": result_iter_time
})
logger.info("[+ +] Completed! All Time:%.5e" % (end_time - start_time))
def pretrain(self,
dataset_train: Dataset,
dataset_vali: Dataset,
max_epoch=50000,
stopping_criterion=1e-4,
lr=1e-3,
model_name="NeuralDynamic.model"):
""" Pre-train the model by using randomly generalized data
Parameters
------------
dataset_train : DynamicModelDataSetWrapper
Data set for training
dataset_vali : DynamicModelDataSetWrapper
Data set for validation
max_epoch : int
Maximum number of epochs if stopping criterion is not reached
stopping_criterion : double
If the objective function of the training set is less than
the stopping criterion, the training is stopped
lr : double
Learning rate
model_name : string
When the stopping criterion,
the model with the given name will be saved as a file
"""
# if the model exists, load the model directly
model_path = logger.logger_path
if model_path is None:
raise Exception(
"Please call logger.set_folder_name(folder_name).set_is_use_logger(True) to enable the logger function!"
)
result_train_loss = np.zeros(max_epoch)
result_vali_loss = np.zeros(int(max_epoch / 100))
if not os.path.exists(os.path.join(model_path, model_name)):
logger.info("[+ +] Model file \"" + model_name +
"\" does NOT exist. Pre-traning starts...")
loss_fun = nn.MSELoss()
# self.optimizer = optim.SGD(self.model.parameters(), lr=lr, momentum=0.9)
optimizer = optim.RAdam(self._model.parameters(),
lr=lr,
weight_decay=1e-4)
X_train, Y_train = self._process_data(dataset_train)
X_vali, Y_vali = self._process_data(dataset_vali)
time_start_pretraining = tm.time()
for epoch in range(max_epoch):
#### Training ###
self._model.train()
optimizer.zero_grad()
Y_prediction = self._model(X_train)
obj_train = loss_fun(Y_prediction, Y_train)
obj_train.backward()
optimizer.step()
result_train_loss[epoch] = obj_train.item()
if obj_train.item(
) < stopping_criterion or epoch % 100 == 0: # Check stopping criterion
## Evaluation ###
self._model.eval()
# Forward Propagation
Y_prediction = self._model(X_vali)
obj_vali = loss_fun(Y_prediction, Y_vali)
##### Print #####
logger.info(
"[+ +] Epoch: %5d Train Loss: %.5e Vali Loss:%.5e"
% (epoch + 1, obj_train.item(), obj_vali.item()))
result_vali_loss[int(np.ceil(epoch /
100))] = obj_vali.item()
if obj_train.item() < stopping_criterion:
time_end_preraining = tm.time()
time_pretraining = time_end_preraining - time_start_pretraining
logger.info(
"[+ +] Pretraining finished! Model file \"" +
model_name + "\" is saved!")
logger.info("[+ +] Pretraining time: %.8f" %
(time_pretraining))
torch.save(self._model.state_dict(),
os.path.join(model_path, model_name))
io.savemat(
os.path.join(model_path,
model_name + "_training.mat"), {
"Train_loss": result_train_loss,
"Vali_loss": result_vali_loss
})
return
raise Exception("Maximum epoch in the pretraining is reached!")
else:
logger.info("[+ +] Model file \"" + model_name +
"\" exists. Loading....")
self._model.load_state_dict(
torch.load(os.path.join(model_path, model_name)))
self._model.eval()