def run(self): #itr_start = 0 #guided_steps = [0.5, 0.4, 0.3, 0.2, 0.1] self.algorithm.policy_opt.policy = TfPolicy.load_policy( policy_dict_path=self.policy_path, tf_generator=fully_connected_tf_network, network_config=self.network_config) #for itr in range(itr_start, self._hyperparams['iterations']): #for m, cond in enumerate(self._train_idx): # for i in range(self._hyperparams['num_samples']): # self._take_sample(itr, cond, m, i) # print('Iteration %d' % (itr)) # traj_sample_lists = [self.agent.get_samples(cond, -self._hyperparams['num_samples']) for cond in self._train_idx] # # Clear agent samples. # self.agent.clear_samples() # self.algorithm.iteration(traj_sample_lists) # # #pol_sample_lists = self._take_policy_samples(self._train_idx) # # #self._prev_traj_costs, self._prev_pol_costs = self.disp.update(itr, self.algorithm, self.agent, traj_sample_lists, pol_sample_lists) # self.algorithm.policy_opt.policy.pickle_policy(self.algorithm.policy_opt._dO, self.algorithm.policy_opt._dU, self._data_files_dir + ('policy_itr_%02d' % itr)) # self._test_peformance(t_length=50) #self.algorithm.policy_opt.policy = TfPolicy.load_policy(policy_dict_path=self.policy_path, tf_generator=fully_connected_tf_network, network_config=self.network_config) self._test_peformance(t_length=50) #pol_sample_lists = self._take_policy_samples(self._test_idx) #self._prev_traj_costs, self._prev_pol_costs = self.disp.update(self.alg orithm, self.agent, self._test_idx, pol_sample_lists) if 'on_exit' in self._hyperparams: self._hyperparams['on_exit'](self._hyperparams)
def main(): #cur_dir = os.path.dirname(os.path.abspath(__file__)) BASE_DIR = '/'.join(str.split(gps_filepath, '/')[:-2]) EXP_DIR = BASE_DIR + '/../experiments/laplace/' network_dir = EXP_DIR + 'data_files_pde/' + ('policy_itr_%02d' % 0) + '.pkl' hyperparams_file = EXP_DIR + 'hyperparams.py' hyperparams = imp.load_source('hyperparams', hyperparams_file).config['algorithm'] #print(network_dir) #print(hyperparams.keys()) pol_dict = pickle.load(open(network_dir, "rb"), encoding='latin1') print(pol_dict.keys(), pol_dict['scale'].shape, pol_dict['bias'].shape) #print(pol_dict['scale']) #print(pol_dict['bias']) network_config = hyperparams['policy_opt']['network_params'] network_config['deg_action'] = 1050 network_config['param_dim'] = network_config['deg_action'] network_config['deg_obs'] = network_config['deg_action'] * ( network_config['history_len'] * 2 + 1) + network_config['history_len'] network = TfPolicy.load_policy(network_dir, first_derivative_network, network_config=network_config) np.random.seed(0) x = np.random.randn(network_config['deg_action']) np.random.seed(0) obs = np.random.randn(network_config['deg_obs']) act = network.act(x, obs, 0, None, usescale=False) print(x.shape, act.shape, obs.shape, act[0:20])
def test_policy_load(): tf_map = POLICY_OPT_TF['network_model'] check_path = gps_path + '/gps/algorithm/policy_opt/tf_checkpoint/policy_checkpoint/_pol' pol = TfPolicy.load_policy(check_path, tf_map) deg_obs = 14 deg_action = 7 N = 20 T = 30 obs = np.random.randn(N, T, deg_obs) obs_reshaped = np.reshape(obs, (N*T, deg_obs)) pol.scale = np.diag(1.0 / np.std(obs_reshaped, axis=0)) pol.bias = -np.mean(obs_reshaped.dot(pol.scale), axis=0) noise = np.random.randn(deg_action) pol.act(None, obs[0, 0], None, None)
def test_policy_load(): tf_map = POLICY_OPT_TF['network_model'] check_path = gps_path + '/gps/algorithm/policy_opt/tf_checkpoint/policy_checkpoint/_pol' pol = TfPolicy.load_policy(check_path, tf_map) deg_obs = 14 deg_action = 7 N = 20 T = 30 obs = np.random.randn(N, T, deg_obs) obs_reshaped = np.reshape(obs, (N * T, deg_obs)) pol.scale = np.diag(1.0 / np.std(obs_reshaped, axis=0)) pol.bias = -np.mean(obs_reshaped.dot(pol.scale), axis=0) noise = np.random.randn(deg_action) pol.act(None, obs[0, 0], None, None)
def pde_policy_comp(hyperparams, exp_dir, pols, input_dim_1=128, input_dim_2=128, a=1.0, b=1.0, learning_rate=0.0001, mem_len=10, momentum=0.9, err=1e-4): dx = a / input_dim_1 dy = b / input_dim_2 x_pts = np.arange(0, a + dx / 2, dx) y_pts = np.arange(0, b + dy / 2, dy) k = np.random.rand(1)[0] + 0.5 #session = tf.Session() session = tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions( allow_growth=True), allow_soft_placement=True)) cost, fcn_family, b = def_cost(dx, dy, x_pts, y_pts, input_dim_1, input_dim_2, k, session) normb = np.linalg.norm(b) obs_flag = hyperparams.config['algorithm']['policy_opt']['network_params'] history_len = hyperparams.config['agent']['history_len'] param_dim = fcn_family.get_total_num_dim() #print(param_dim) init_loc = np.random.rand(param_dim, 1) fcns = [{'fcn_obj': cost, 'dim': param_dim, 'init_loc': init_loc}] SENSOR_DIMS = { CUR_LOC: param_dim, PAST_OBJ_VAL_DELTAS: history_len, PAST_GRADS: history_len * param_dim, PAST_LOC_DELTAS: history_len * param_dim, CUR_GRAD: param_dim, ACTION: param_dim } obs_flag['sensor_dims'] = SENSOR_DIMS obs_flag['param_dim'] = param_dim agent = { 'substeps': hyperparams.config['agent']['substeps'], 'conditions': 1, 'dt': hyperparams.config['agent']['dt'], 'T': hyperparams.config['agent']['T'], 'sensor_dims': SENSOR_DIMS, 'state_include': hyperparams.config['agent']['state_include'], 'obs_include': hyperparams.config['agent']['obs_include'], 'history_len': history_len, 'fcns': fcns, 'fcn_family': fcn_family } network_config = hyperparams.config['algorithm']['policy_opt'][ 'network_params'] network_config['deg_action'] = param_dim network_config['param_dim'] = network_config['deg_action'] network_config['deg_obs'] = int( np.sum([SENSOR_DIMS[sensor] for sensor in agent['obs_include']])) print("****************************************************************") print('Initial relative error:', np.sqrt(cost.evaluate(fcns[0]['init_loc'])) / normb) gd_fcns = [{'fcn_obj': cost, 'dim': param_dim, 'init_loc': init_loc}] cg_fcns = [{'fcn_obj': cost, 'dim': param_dim, 'init_loc': init_loc}] lbfgs_fcns = [{'fcn_obj': cost, 'dim': param_dim, 'init_loc': init_loc}] mm_fcns = [{'fcn_obj': cost, 'dim': param_dim, 'init_loc': init_loc}] lr_fcns = [{'fcn_obj': cost, 'dim': param_dim, 'init_loc': init_loc}] #agent_gd = copy.deepcopy(agent) #agent_cg = copy.deepcopy(agent) #agent_lbfgs = copy.deepcopy(agent) #agent_mm = copy.deepcopy(agent) #agent_lr = copy.deepcopy(agent) for i in range(len(pols)): agent['fcns'] = gd_fcns Agent_gd = AgentLTO(agent) gd_pol = GradientDescentPolicy(Agent_gd, learning_rate, 0) agent['fcns'] = cg_fcns Agent_cg = AgentLTO(agent) cg_pol = ConjugateGradientPolicy(Agent_cg, learning_rate, 0) agent['fcns'] = lbfgs_fcns Agent_lbfgs = AgentLTO(agent) lbfgs_pol = LBFGSPolicy(Agent_lbfgs, learning_rate, mem_len, 0) agent['fcns'] = mm_fcns Agent_mm = AgentLTO(agent) mm_pol = MomentumPolicy(Agent_mm, learning_rate, momentum, 0) agent['fcns'] = lr_fcns Agent_lr = AgentLTO(agent) network_dir = exp_dir + 'data_files_pde/' + ('policy_itr_%02d' % pols[i]) + '.pkl' lr_pol = TfPolicy.load_policy(network_dir, first_derivative_network, network_config=network_config) x_gd = np.expand_dims(Agent_gd.sample(gd_pol, 0, verbose=False, save=False, noisy=False, usescale=False).get_X()[-1], axis=1) gd_fcns[0]['init_loc'] = x_gd print('Relative error after', agent['T'], 'iteration using GradientDescent Policy :', np.sqrt(cost.evaluate(x_gd)) / normb) x_cg = np.expand_dims(Agent_cg.sample(cg_pol, 0, verbose=False, save=False, noisy=False, usescale=False).get_X()[-1], axis=1) cg_fcns[0]['init_loc'] = x_cg print('Relative error after', agent['T'], 'iteration using ConjuageGradient Policy:', np.sqrt(cost.evaluate(x_cg)) / normb) x_lbfgs = np.expand_dims(Agent_lbfgs.sample( lbfgs_pol, 0, verbose=False, save=False, noisy=False, usescale=False).get_X()[-1], axis=1) lbfgs_fcns[0]['init_loc'] = x_lbfgs print('Relative error after', agent['T'], 'iteration using LBFGS Policy :', np.sqrt(cost.evaluate(x_lbfgs)) / normb) x_mm = np.expand_dims(Agent_mm.sample(mm_pol, 0, verbose=False, save=False, noisy=False, usescale=False).get_X()[-1], axis=1) mm_fcns[0]['init_loc'] = x_mm print('Relative error after', agent['T'], 'iteration using Momentum Policy :', np.sqrt(cost.evaluate(x_mm)) / normb) x_lr = np.expand_dims(Agent_lr.sample(lr_pol, 0, verbose=False, save=False, noisy=False, usescale=False).get_X()[-1], axis=1) lr_fcns[0]['init_loc'] = x_lr print('Relative error after', agent['T'], 'iteration using Learned Policy :', np.sqrt(cost.evaluate(x_lr)) / normb)