# -----------invser functions------------- from InverseFuncs import trajectory, getLoss, reset_theta, theta_range,reset_theta_log, single_theta_inverse # ---------loading env and agent---------- from stable_baselines import TD3 from FireflyEnv import firefly_accac from Config import Config arg = Config() DISCOUNT_FACTOR = 0.99 arg.NUM_SAMPLES=2 arg.NUM_EP = 200 arg.NUM_IT = 2 # number of iteration for gradient descent arg.NUM_thetas = 1 arg.ADAM_LR = 0.25 arg.LR_STEP = 50 arg.LR_STOP = 0.1 arg.lr_gamma = 0.95 arg.PI_STD=1 arg.goal_radius_range=[0.1,0.3] arg.TERMINAL_VEL = 0.025 arg.goal_radius_range=[0.15,0.3] arg.std_range = [0.02,0.3,0.02,0.3] arg.TERMINAL_VEL = 0.025 # terminal velocity? # norm(action) that you believe as a signal to stop 0.1. arg.DELTA_T=0.2 arg.EPISODE_LEN=35 number_updates=100 # agent convert to torch model
arg.mag_action_cost_range= [0.0001,0.001]
arg.dev_action_cost_range= [0.0001,0.005]
arg.dev_v_cost_range= [0.1,0.5]
arg.dev_w_cost_range= [0.1,0.5]
arg.TERMINAL_VEL = 0.1
arg.DELTA_T=0.1
arg.EPISODE_LEN=100
arg.agent_knows_phi=False
DISCOUNT_FACTOR = 0.99
arg.sample=100
arg.batch = 70
# arg.NUM_SAMPLES=1
# arg.NUM_EP=1
arg.NUM_IT = 1
arg.NUM_thetas = 1
arg.ADAM_LR = 0.0002
arg.LR_STEP = 20
arg.LR_STOP = 0.5
arg.lr_gamma = 0.95
arg.PI_STD=1
arg.presist_phi=False
arg.cost_scale=1
number_updates=10000
# load torch model
import TD3_torch
# agent=TD3_torch.TD3.load('trained_agent/500000_1_9_21_8.zip')
# agent=agent.actor.mu.cpu()
agent_ =SAC.load('trained_agent/slowrewardnoskp_200000_1_7_22_12.zip')
agent_=agent_.actor.cpu()
agent = lambda x : agent_.forward(x, deterministic=True)
def run_inverse(data=None,theta=None,filename=None):
import os
import warnings
warnings.filterwarnings('ignore')
from copy import copy
import time
import random
seed=time.time().as_integer_ratio()[0]
seed=0
random.seed(seed)
import torch
torch.manual_seed(seed)
import numpy as np
np.random.seed(int(seed))
from numpy import pi
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
# -----------invser functions-------------
from InverseFuncs import trajectory, getLoss, reset_theta, theta_range,reset_theta_log, single_inverse
# ---------loading env and agent----------
from stable_baselines import DDPG,TD3
from FireflyEnv import ffenv_new_cord
from Config import Config
arg = Config()
DISCOUNT_FACTOR = 0.99
arg.NUM_SAMPLES=2
arg.NUM_EP = 1000
arg.NUM_IT = 2 # number of iteration for gradient descent
arg.NUM_thetas = 1
arg.ADAM_LR = 0.007
arg.LR_STEP = 2
arg.LR_STOP = 50
arg.lr_gamma = 0.95
arg.PI_STD=1
arg.goal_radius_range=[0.05,0.2]
# agent convert to torch model
import policy_torch
baselines_mlp_model = TD3.load('trained_agent//TD_95gamma_mc_smallgoal_500000_9_24_1_6.zip')
agent = policy_torch.copy_mlp_weights(baselines_mlp_model,layers=[128,128])
# loading enviorment, same as training
env=ffenv_new_cord.FireflyAgentCenter(arg)
env.agent_knows_phi=False
true_theta_log = []
true_loss_log = []
true_loss_act_log = []
true_loss_obs_log = []
final_theta_log = []
stderr_log = []
result_log = []
number_update=100
if data is None:
save_dict={'theta_estimations':[]}
else:
save_dict=data
# use serval theta to inverse
for num_thetas in range(arg.NUM_thetas):
# make sure phi and true theta stay the same
true_theta = torch.Tensor(data['true_theta'])
env.presist_phi=True
env.reset(phi=true_theta,theta=true_theta) # here we first testing teacher truetheta=phi case
theta=torch.Tensor(data['theta_estimations'][0])
phi=torch.Tensor(data['phi'])
save_dict['true_theta']=true_theta.data.clone().tolist()
save_dict['phi']=true_theta.data.clone().tolist()
save_dict['inital_theta']=theta.data.clone().tolist()
for num_update in range(number_update):
states, actions, tasks = trajectory(
agent, phi, true_theta, env, arg.NUM_EP)
result = single_theta_inverse(true_theta, phi, arg, env, agent, states, actions, tasks, filename, num_thetas, initial_theta=theta)
save_dict['theta_estimations'].append(result.tolist())
if filename is None:
savename=('inverse_data/' + filename + "EP" + str(arg.NUM_EP) + "updates" + str(number_update)+"sample"+str(arg.NUM_SAMPLES) +"IT"+ str(arg.NUM_IT) + '.pkl')
torch.save(save_dict, savename)
elif filename[:-4]=='.pkl':
torch.save(save_dict, filename)
else:
torch.save(save_dict, (filename+'.pkf'))
print(result)
print('done')
# -----------invser functions-------------
from InverseFuncs import trajectory, getLoss, reset_theta, theta_range,reset_theta_log, single_theta_inverse
# ---------loading env and agent----------
from stable_baselines import DDPG,TD3
from FireflyEnv import ffenv_new_cord
from Config import Config
arg = Config()
DISCOUNT_FACTOR = 0.99
arg.NUM_SAMPLES=2
arg.NUM_EP = 100
arg.NUM_IT = 2 # number of iteration for gradient descent
arg.NUM_thetas = 1
arg.ADAM_LR = 0.07
arg.LR_STEP = 2
arg.LR_STOP = 0.0001
arg.lr_gamma = 0.95
arg.PI_STD=1
arg.goal_radius_range=[0.05,0.2]
number_updates=500
# agent convert to torch model
import policy_torch
baselines_mlp_model = TD3.load('trained_agent//TD_95gamma_mc_smallgoal_500000_9_24_1_6.zip')
agent = policy_torch.copy_mlp_weights(baselines_mlp_model,layers=[128,128])
# loading enviorment, same as training
env=ffenv_new_cord.FireflyAgentCenter(arg)
# ---seting the env for inverse----
