def unc_premodel(env, env_name, model_name): path = './uncertainty_modeling/rl_uncertainty' obs_dim = env.observation_space.low.size action_dim = env.action_space.low.size input_size = obs_dim + action_dim model = None if model_name == 'mc_dropout': model = FlattenMlp_Dropout( # Check the dropout layer! input_size=input_size, output_size=1, hidden_sizes=[256, 256], ).cuda() if model_name == 'rank1': model = Model(x_dim=input_size, h_dim=10, y_dim=1, n=10).cuda() if model_name == 'swag': kwargs = {"dimensions": [200, 50, 50, 50], "output_dim": 1, "input_dim": input_size} args = list() model = SWAG(RegNetBase, subspace_type="pca", *args, **kwargs, subspace_kwargs={"max_rank": 10, "pca_rank": 10}) model.cuda() if model == None: raise AttributeError else: model.load_state_dict(torch.load('{}/{}/model/{}/model_200.pt'.format(path, model_name, env_name))) if model_name == 'swag': model.sample(scale=10.) return model
def test(): env = gym.make(opts.env_name) obs_dim = env.observation_space.low.size action_dim = env.action_space.low.size input_size = obs_dim + action_dim ## Choose the trained model dataloader = DataLoader( # ScatterDataset(path='reg_data/test_data.npy'), GymDataset(env, opts.ood_test, opts.env_name), batch_size=400, shuffle=True, num_workers=8, ) ## Choose the training model model = FlattenMlp_Dropout( input_size=input_size, output_size=1, hidden_sizes=[256, 256], ).cuda() model.load_state_dict( torch.load("{}/{}/model_100.pt".format( path, opts.env_name))) # if not handling ensemble for i, data in enumerate(dataloader): id_obs_act = Variable(data['id_obs_act'].type(Tensor)) ood_obs_act = Variable(data['ood_obs_act'].type(Tensor)) # if i == 0 : with torch.no_grad(): ## Load testing dataset id_trajectories, ood_trajectories = [], [] ## Iterative test for each model for i in range(10): id_output_ = model(id_obs_act).cpu().numpy().T ood_output_ = model(ood_obs_act).cpu().numpy().T id_trajectories.append(id_output_[:1, :]) ood_trajectories.append(ood_output_[:1, :]) id_trajectories = np.vstack(id_trajectories) ood_trajectories = np.vstack(ood_trajectories) # id_sigma = np.std(id_trajectories, axis=0) # ood_sigma = np.std(ood_trajectories, axis=0) id_sigma = np.mean(id_trajectories**2, axis=0) - np.mean( id_trajectories, axis=0)**2 ood_sigma = np.mean(ood_trajectories**2, axis=0) - np.mean( ood_trajectories, axis=0)**2 print('id_sigma : {}, ood_sigma : {}'.format( np.mean(id_sigma), np.mean(ood_sigma)))
def test(): ## Choose the trained model model = FlattenMlp_Dropout( input_size=23, output_size=1, hidden_sizes=[128, 128], ) dataloader = DataLoader( GymDataset_test(), batch_size=1000, shuffle=True, num_workers=8, ) model.load_state_dict(torch.load("./dropout_128/rl_dropout_" + str(60) + ".pt")) # if not handling ensemble for i, data in enumerate(dataloader): id_obs_act = Variable(data['id_obs_act'].type(Tensor)) ood_obs_act = Variable(data['ood_obs_act'].type(Tensor)) # if i == 0 : with torch.no_grad(): ## Load testing dataset id_trajectories, ood_trajectories = [], [] ## Iterative test for each model for i in range(10): id_output_ = model(id_obs_act).cpu().numpy().T ood_output_ = model(ood_obs_act).cpu().numpy().T id_trajectories.append(id_output_[:1, :]) ood_trajectories.append(ood_output_[:1, :]) id_trajectories = np.vstack(id_trajectories) ood_trajectories = np.vstack(ood_trajectories) # id_sigma = np.std(id_trajectories, axis=0) # ood_sigma = np.std(ood_trajectories, axis=0) id_sigma = np.mean(id_trajectories**2, axis=0) - np.mean(id_trajectories, axis=0) ** 2 ood_sigma = np.mean(ood_trajectories ** 2, axis=0) - np.mean(ood_trajectories, axis=0) ** 2 print('id_sigma : {}, ood_sigma : {}'.format(np.mean(id_sigma), np.mean(ood_sigma)))
import torch import torch.optim as optim from torch import nn as nn from rlkit.torch.networks import FlattenMlp_Dropout import rlkit.torch.pytorch_util as ptu from rlkit.core.eval_util import create_stats_ordered_dict from rlkit.torch.torch_rl_algorithm import TorchTrainer from torch import autograd model = FlattenMlp_Dropout( input_size=23, output_size=1, hidden_sizes=[256, 256], ).cuda() model.load_state_dict( torch.load( '/home/user/Documents/Workspace-Changyeop/Workspace/AdvancedDL/AI602_Project/bear/rlkit/torch/sac/rl_dropout_140.pt' )) def uncertainty(state, action, rep, beta): with torch.no_grad(): batch_size = state.shape[0] state_cp = state.unsqueeze(1).repeat(1, rep, 1).view(state.shape[0] * rep, state.shape[1]) action_cp = action.unsqueeze(1).repeat(1, rep, 1).view(action.shape[0] * rep, action.shape[1]) target_qf1 = model(torch.cat([state_cp, action_cp], dim=1)) # BTx1 target_qf1 = target_qf1.view(batch_size, rep, 1) # BxTx1