# -----------invser functions------------- from InverseFuncs import trajectory, getLoss, reset_theta, theta_range,reset_theta_log, single_theta_inverse # ---------loading env and agent---------- from stable_baselines import TD3 from FireflyEnv import firefly_accac from Config import Config arg = Config() DISCOUNT_FACTOR = 0.99 arg.NUM_SAMPLES=2 arg.NUM_EP = 200 arg.NUM_IT = 2 # number of iteration for gradient descent arg.NUM_thetas = 1 arg.ADAM_LR = 0.25 arg.LR_STEP = 50 arg.LR_STOP = 0.1 arg.lr_gamma = 0.95 arg.PI_STD=1 arg.goal_radius_range=[0.1,0.3] arg.TERMINAL_VEL = 0.025 arg.goal_radius_range=[0.15,0.3] arg.std_range = [0.02,0.3,0.02,0.3] arg.TERMINAL_VEL = 0.025 # terminal velocity? # norm(action) that you believe as a signal to stop 0.1. arg.DELTA_T=0.2 arg.EPISODE_LEN=35 number_updates=100 # agent convert to torch model
arg.mag_action_cost_range= [0.0001,0.001] arg.dev_action_cost_range= [0.0001,0.005] arg.dev_v_cost_range= [0.1,0.5] arg.dev_w_cost_range= [0.1,0.5] arg.TERMINAL_VEL = 0.1 arg.DELTA_T=0.1 arg.EPISODE_LEN=100 arg.agent_knows_phi=False DISCOUNT_FACTOR = 0.99 arg.sample=100 arg.batch = 70 # arg.NUM_SAMPLES=1 # arg.NUM_EP=1 arg.NUM_IT = 1 arg.NUM_thetas = 1 arg.ADAM_LR = 0.0002 arg.LR_STEP = 20 arg.LR_STOP = 0.5 arg.lr_gamma = 0.95 arg.PI_STD=1 arg.presist_phi=False arg.cost_scale=1 number_updates=10000 # load torch model import TD3_torch # agent=TD3_torch.TD3.load('trained_agent/500000_1_9_21_8.zip') # agent=agent.actor.mu.cpu() agent_ =SAC.load('trained_agent/slowrewardnoskp_200000_1_7_22_12.zip') agent_=agent_.actor.cpu() agent = lambda x : agent_.forward(x, deterministic=True)
def run_inverse(data=None,theta=None,filename=None): import os import warnings warnings.filterwarnings('ignore') from copy import copy import time import random seed=time.time().as_integer_ratio()[0] seed=0 random.seed(seed) import torch torch.manual_seed(seed) import numpy as np np.random.seed(int(seed)) from numpy import pi torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False # -----------invser functions------------- from InverseFuncs import trajectory, getLoss, reset_theta, theta_range,reset_theta_log, single_inverse # ---------loading env and agent---------- from stable_baselines import DDPG,TD3 from FireflyEnv import ffenv_new_cord from Config import Config arg = Config() DISCOUNT_FACTOR = 0.99 arg.NUM_SAMPLES=2 arg.NUM_EP = 1000 arg.NUM_IT = 2 # number of iteration for gradient descent arg.NUM_thetas = 1 arg.ADAM_LR = 0.007 arg.LR_STEP = 2 arg.LR_STOP = 50 arg.lr_gamma = 0.95 arg.PI_STD=1 arg.goal_radius_range=[0.05,0.2] # agent convert to torch model import policy_torch baselines_mlp_model = TD3.load('trained_agent//TD_95gamma_mc_smallgoal_500000_9_24_1_6.zip') agent = policy_torch.copy_mlp_weights(baselines_mlp_model,layers=[128,128]) # loading enviorment, same as training env=ffenv_new_cord.FireflyAgentCenter(arg) env.agent_knows_phi=False true_theta_log = [] true_loss_log = [] true_loss_act_log = [] true_loss_obs_log = [] final_theta_log = [] stderr_log = [] result_log = [] number_update=100 if data is None: save_dict={'theta_estimations':[]} else: save_dict=data # use serval theta to inverse for num_thetas in range(arg.NUM_thetas): # make sure phi and true theta stay the same true_theta = torch.Tensor(data['true_theta']) env.presist_phi=True env.reset(phi=true_theta,theta=true_theta) # here we first testing teacher truetheta=phi case theta=torch.Tensor(data['theta_estimations'][0]) phi=torch.Tensor(data['phi']) save_dict['true_theta']=true_theta.data.clone().tolist() save_dict['phi']=true_theta.data.clone().tolist() save_dict['inital_theta']=theta.data.clone().tolist() for num_update in range(number_update): states, actions, tasks = trajectory( agent, phi, true_theta, env, arg.NUM_EP) result = single_theta_inverse(true_theta, phi, arg, env, agent, states, actions, tasks, filename, num_thetas, initial_theta=theta) save_dict['theta_estimations'].append(result.tolist()) if filename is None: savename=('inverse_data/' + filename + "EP" + str(arg.NUM_EP) + "updates" + str(number_update)+"sample"+str(arg.NUM_SAMPLES) +"IT"+ str(arg.NUM_IT) + '.pkl') torch.save(save_dict, savename) elif filename[:-4]=='.pkl': torch.save(save_dict, filename) else: torch.save(save_dict, (filename+'.pkf')) print(result) print('done')
# -----------invser functions------------- from InverseFuncs import trajectory, getLoss, reset_theta, theta_range,reset_theta_log, single_theta_inverse # ---------loading env and agent---------- from stable_baselines import DDPG,TD3 from FireflyEnv import ffenv_new_cord from Config import Config arg = Config() DISCOUNT_FACTOR = 0.99 arg.NUM_SAMPLES=2 arg.NUM_EP = 100 arg.NUM_IT = 2 # number of iteration for gradient descent arg.NUM_thetas = 1 arg.ADAM_LR = 0.07 arg.LR_STEP = 2 arg.LR_STOP = 0.0001 arg.lr_gamma = 0.95 arg.PI_STD=1 arg.goal_radius_range=[0.05,0.2] number_updates=500 # agent convert to torch model import policy_torch baselines_mlp_model = TD3.load('trained_agent//TD_95gamma_mc_smallgoal_500000_9_24_1_6.zip') agent = policy_torch.copy_mlp_weights(baselines_mlp_model,layers=[128,128]) # loading enviorment, same as training env=ffenv_new_cord.FireflyAgentCenter(arg) # ---seting the env for inverse----