def solve(self): """ Solve the problem with classical iLQR """ # Initialize the trajectory, F_matrix, objective_function_value_last, C_matrix and c_vector self._trajectory = self._dynamic_model.eval_traj( ) # init feasible trajectory C_matrix = self._obj_fun.eval_hessian_obj_fun(self._trajectory) c_vector = self._obj_fun.eval_grad_obj_fun(self._trajectory) F_matrix = self._dynamic_model.eval_grad_dynamic_model( self._trajectory) logger.info("[+ +] Initial Obj.Val.: %.5e" % (self._obj_fun.eval_obj_fun(self._trajectory))) # Start iteration start_time = tm.time() for i in range(self._max_iter): if i == 1: # skip the compiling time start_time = tm.time() iter_start_time = tm.time() K_matrix, k_vector = self.backward_pass(C_matrix, c_vector, F_matrix) backward_time = tm.time() self._trajectory, C_matrix, c_vector, F_matrix, obj, isStop = self.forward_pass( self._trajectory, K_matrix, k_vector) forward_time = tm.time() logger.info( "[+ +] Iter.No.%3d BWTime:%.3e FWTime:%.3e Obj.Val.:%.5e" % (i, backward_time - iter_start_time, forward_time - backward_time, obj)) logger.save_to_json(trajectory=self._trajectory.tolist()) if isStop and self._is_check_stop: break end_time = tm.time() logger.info("[+ +] Completed! All Time:%.5e" % (end_time - start_time))
def retrain(self, dataset, max_epoch=10000, stopping_criterion=1e-3, lr=1e-3): logger.info("[+ +] Re-traning starts...") loss_fun = nn.MSELoss() # optimizer = optim.SGD(self.model.parameters(), lr=lr, momentum=0.9) optimizer = optim.RAdam(self._model.parameters(), lr=lr, weight_decay=1e-4) X_train, Y_train = self._process_data(dataset) for epoch in range(max_epoch): #### Training ### self._model.train() optimizer.zero_grad() Y_prediction = self._model(X_train) obj_train = loss_fun(Y_prediction, Y_train) obj_train.backward() optimizer.step() if obj_train.item( ) < stopping_criterion or epoch % 100 == 0: # Check stopping criterion logger.info("[+ +] Epoch: %5d Train Obj: %.5e" % (epoch + 1, obj_train.item())) if obj_train.item() < stopping_criterion: logger.info("[+ +] Re-training finished!") self._model.eval() return raise Exception("Maximum epoch in the retraining is reached!")
def solve(self): """ Solve the problem with classical iLQR """ # Initialize the trajectory, F_matrix, objective_function_value_last, C_matrix and c_vector self._trajectory = self._dynamic_model.eval_traj() # init feasible trajectory C_matrix = self._obj_fun.eval_hessian_obj_fun(self._trajectory) c_vector = self._obj_fun.eval_grad_obj_fun(self._trajectory) F_matrix = self._dynamic_model.eval_grad_dynamic_model( self._trajectory) # Start iteration logger.info("[+ +] Initial Obj.Val.: %.5e" % (self._real_obj_fun.eval_obj_fun(self._trajectory))) total_iter_no = -1 for j in self._t: if j != self._t[0]: # update t parameter add_param = self.get_obj_add_param() add_param[:, -1] = j*np.ones((self.dynamic_model._T)) self.set_obj_add_param(add_param) for i in range(self._max_iter): total_iter_no += 1 if j == self._t[0] and i == 1: # skip the compiling time start_time = tm.time() iter_start_time = tm.time() K_matrix, k_vector = self.backward_pass( C_matrix, c_vector, F_matrix) backward_time = tm.time() self._trajectory, C_matrix, c_vector, F_matrix, _, isStop = self.forward_pass( self._trajectory, K_matrix, k_vector) forward_time = tm.time() # do not care the value of log barrier obj = self._real_obj_fun.eval_obj_fun(self._trajectory) logger.info("[+ +] Total Iter.No.%3d Iter.No.%3d BWTime:%.3e FWTime:%.3e Obj.Val.:%.5e" % ( total_iter_no, i, backward_time-iter_start_time, forward_time-backward_time, obj)) logger.save_to_json(trajectory=self._trajectory.tolist()) if isStop and self._is_check_stop: self.set_obj_fun_value(np.inf) logger.info( "[+ +] Complete One Inner Loop! The log barrier parameter t is %.5f" % (j) + " in this iteration!") break end_time = tm.time() logger.info("[+ +] Completed! All Time:%.5e" % (end_time-start_time))
def print_params(self): for key in self.kwargs: logger.info("[+] " + key + " = " + str(self.kwargs[key]))
def solve(self): trajectory = self._dynamic_model.eval_traj( ) # init feasible trajectory init_obj = self._obj_fun.eval_obj_fun(trajectory) logger.info("[+ +] Initial Obj.Val.: %.5e" % (init_obj)) self.set_obj_fun_value(init_obj) new_data = [] result_obj_val = np.zeros(self._iLQR_max_iter) result_iter_time = np.zeros(self._iLQR_max_iter) re_train_stopping_criterion = self._training_stopping_criterion for i in range(int(self._iLQR_max_iter)): if i == 1: # skip the compiling time start_time = tm.time() iter_start_time = tm.time() C_matrix = self._obj_fun.eval_hessian_obj_fun(trajectory) c_vector = self._obj_fun.eval_grad_obj_fun(trajectory) F_matrix = self._nn_dynamic_model.eval_grad_dynamic_model( trajectory) F_matrix = gaussian_filter1d(F_matrix, sigma=self._gaussian_filter_sigma, axis=0) K_matrix, k_vector = self.backward_pass(C_matrix, c_vector, F_matrix) trajectory, C_matrix, c_vector, F_matrix, obj_val, isStop = self.forward_pass( trajectory, K_matrix, k_vector) if i < self._decay_rate_max_iters: re_train_stopping_criterion = re_train_stopping_criterion * self._decay_rate iter_end_time = tm.time() iter_time = iter_end_time - iter_start_time logger.info("[+ +] Iter.No.:%3d Iter.Time:%.3e Obj.Val.:%.5e" % ( i, iter_time, obj_val, )) result_obj_val[i] = obj_val result_iter_time[i] = iter_time if isStop: if len( new_data ) != 0: # Ensure the optimal trajectroy being in the dataset trajectory_noisy = trajectory else: trajectory_noisy = self.dynamic_model.eval_traj( action_traj=( trajectory[:, self.dynamic_model.n:] + np.random.normal(0, self._gaussian_noise_sigma, [ self.dynamic_model.T, self.dynamic_model.m, 1 ]))) new_data += [trajectory_noisy] data = np.concatenate( new_data[-int(self._trial_no / 5):])[:, :, 0] self._dataset_train.update_dataset(Data(state=data)) logger.save_to_json(trajectory=trajectory.tolist(), trajectroy_noisy=trajectory_noisy.tolist()) self._nn_dynamic_model.retrain( self._dataset_train, max_epoch=100000, stopping_criterion=re_train_stopping_criterion) new_data = [] else: new_data += [trajectory] end_time = tm.time() io.savemat(os.path.join(logger.logger_path, "_result.mat"), { "obj_val": result_obj_val, "iter_time": result_iter_time }) logger.info("[+ +] Completed! All Time:%.5e" % (end_time - start_time))
def pretrain(self, dataset_train: Dataset, dataset_vali: Dataset, max_epoch=50000, stopping_criterion=1e-4, lr=1e-3, model_name="NeuralDynamic.model"): """ Pre-train the model by using randomly generalized data Parameters ------------ dataset_train : DynamicModelDataSetWrapper Data set for training dataset_vali : DynamicModelDataSetWrapper Data set for validation max_epoch : int Maximum number of epochs if stopping criterion is not reached stopping_criterion : double If the objective function of the training set is less than the stopping criterion, the training is stopped lr : double Learning rate model_name : string When the stopping criterion, the model with the given name will be saved as a file """ # if the model exists, load the model directly model_path = logger.logger_path if model_path is None: raise Exception( "Please call logger.set_folder_name(folder_name).set_is_use_logger(True) to enable the logger function!" ) result_train_loss = np.zeros(max_epoch) result_vali_loss = np.zeros(int(max_epoch / 100)) if not os.path.exists(os.path.join(model_path, model_name)): logger.info("[+ +] Model file \"" + model_name + "\" does NOT exist. Pre-traning starts...") loss_fun = nn.MSELoss() # self.optimizer = optim.SGD(self.model.parameters(), lr=lr, momentum=0.9) optimizer = optim.RAdam(self._model.parameters(), lr=lr, weight_decay=1e-4) X_train, Y_train = self._process_data(dataset_train) X_vali, Y_vali = self._process_data(dataset_vali) time_start_pretraining = tm.time() for epoch in range(max_epoch): #### Training ### self._model.train() optimizer.zero_grad() Y_prediction = self._model(X_train) obj_train = loss_fun(Y_prediction, Y_train) obj_train.backward() optimizer.step() result_train_loss[epoch] = obj_train.item() if obj_train.item( ) < stopping_criterion or epoch % 100 == 0: # Check stopping criterion ## Evaluation ### self._model.eval() # Forward Propagation Y_prediction = self._model(X_vali) obj_vali = loss_fun(Y_prediction, Y_vali) ##### Print ##### logger.info( "[+ +] Epoch: %5d Train Loss: %.5e Vali Loss:%.5e" % (epoch + 1, obj_train.item(), obj_vali.item())) result_vali_loss[int(np.ceil(epoch / 100))] = obj_vali.item() if obj_train.item() < stopping_criterion: time_end_preraining = tm.time() time_pretraining = time_end_preraining - time_start_pretraining logger.info( "[+ +] Pretraining finished! Model file \"" + model_name + "\" is saved!") logger.info("[+ +] Pretraining time: %.8f" % (time_pretraining)) torch.save(self._model.state_dict(), os.path.join(model_path, model_name)) io.savemat( os.path.join(model_path, model_name + "_training.mat"), { "Train_loss": result_train_loss, "Vali_loss": result_vali_loss }) return raise Exception("Maximum epoch in the pretraining is reached!") else: logger.info("[+ +] Model file \"" + model_name + "\" exists. Loading....") self._model.load_state_dict( torch.load(os.path.join(model_path, model_name))) self._model.eval()