def test_multivariate_state_std_gaussian(): np.random.seed(88) n_dims = 5 n_outs = 3 mu_approximator = Regressor(LinearApproximator, input_shape=(n_dims, ), output_shape=(n_outs, )) std_approximator = Regressor(LinearApproximator, input_shape=(n_dims, ), output_shape=(n_outs, )) pi = StateStdGaussianPolicy(mu_approximator, std_approximator) weights = np.random.rand(pi.weights_size) + .1 pi.set_weights(weights) x = np.random.randn(20, n_dims) for x_i in x: state = np.atleast_1d(x_i) action = pi.draw_action(state) exact_diff = pi.diff(state, action) numerical_diff = numerical_diff_policy(pi, state, action) assert np.allclose(exact_diff, numerical_diff)
def multivariate_state_std_gaussian(): print('Testing multivariate state std gaussian policy...') n_dims = 5 n_outs = 3 std = np.random.randn(n_outs) approximator_params = dict(input_dim=n_dims) mu_approximator = Regressor(LinearApproximator, input_shape=(n_dims,), output_shape=(n_outs,), params=approximator_params) std_approximator = Regressor(LinearApproximator, input_shape=(n_dims,), output_shape=(n_outs,), params=approximator_params) pi = StateStdGaussianPolicy(mu_approximator, std_approximator) mu_weights = np.random.rand(pi.weights_size)+0.1 pi.set_weights(mu_weights) x = np.random.randn(20, n_dims) for x_i in x: state = np.atleast_1d(x_i) action = pi.draw_action(state) exact_diff = pi.diff(state, action) numerical_diff = numerical_diff_policy(pi, state, action) assert np.allclose(exact_diff, numerical_diff)
def __init__(self, approximator, policy, mdp_info, batch_size, target_update_frequency, initial_replay_size, max_replay_size, fit_params=None, approximator_params=None, n_approximators=1, clip_reward=True, weighted_update=False, update_type='weighted', delta=0.1, q_max=100, store_prob=False, max_spread=None): self._fit_params = dict() if fit_params is None else fit_params self._batch_size = batch_size self._n_approximators = n_approximators self._clip_reward = clip_reward self._target_update_frequency = target_update_frequency self.weighted_update = weighted_update self.update_type = update_type self.q_max = q_max self.store_prob = store_prob self.max_spread = max_spread quantiles = [ i * 1. / (n_approximators - 1) for i in range(n_approximators) ] for p in range(n_approximators): if quantiles[p] >= 1 - delta: self.delta_index = p break self._replay_memory = ReplayMemory(initial_replay_size, max_replay_size) self._n_updates = 0 apprx_params_train = deepcopy(approximator_params) apprx_params_train['name'] = 'train' apprx_params_target = deepcopy(approximator_params) apprx_params_target['name'] = 'target' self.approximator = Regressor(approximator, **apprx_params_train) self.target_approximator = Regressor(approximator, **apprx_params_target) policy.set_q(self.approximator) self.target_approximator.model.set_weights( self.approximator.model.get_weights()) super(ParticleDQN, self).__init__(policy, mdp_info)
def __init__(self, approximator, policy, mdp_info, batch_size, initial_replay_size, max_replay_size, approximator_params, target_update_frequency, fit_params=None, n_approximators=1, clip_reward=True): """ Constructor. Args: approximator (object): the approximator to use to fit the Q-function; batch_size (int): the number of samples in a batch; initial_replay_size (int): the number of samples to collect before starting the learning; max_replay_size (int): the maximum number of samples in the replay memory; approximator_params (dict): parameters of the approximator to build; target_update_frequency (int): the number of samples collected between each update of the target network; fit_params (dict, None): parameters of the fitting algorithm of the approximator; n_approximators (int, 1): the number of approximator to use in ``AverageDQN``; clip_reward (bool, True): whether to clip the reward or not. """ self._fit_params = dict() if fit_params is None else fit_params self._batch_size = batch_size self._n_approximators = n_approximators self._clip_reward = clip_reward self._target_update_frequency = target_update_frequency self._replay_memory = ReplayMemory(initial_replay_size, max_replay_size) self._n_updates = 0 apprx_params_train = deepcopy(approximator_params) apprx_params_target = deepcopy(approximator_params) self.approximator = Regressor(approximator, **apprx_params_train) self.target_approximator = Regressor(approximator, n_models=self._n_approximators, **apprx_params_target) policy.set_q(self.approximator) if self._n_approximators == 1: self.target_approximator.model.set_weights( self.approximator.model.get_weights()) else: for i in range(self._n_approximators): self.target_approximator.model[i].set_weights( self.approximator.model.get_weights()) super().__init__(policy, mdp_info)
def __init__(self, approximator, policy, mdp_info, batch_size, target_update_frequency, initial_replay_size, train_frequency, max_replay_size, fit_params=None, approximator_params=None, n_approximators=1, history_length=1, clip_reward=True, max_no_op_actions=0, no_op_action_value=0, p_mask=2 / 3., dtype=np.float32, weighted_update=False): self._fit_params = dict() if fit_params is None else fit_params self._batch_size = batch_size self._n_approximators = n_approximators self._clip_reward = clip_reward self._target_update_frequency = target_update_frequency // train_frequency self._max_no_op_actions = max_no_op_actions self._no_op_action_value = no_op_action_value self._p_mask = p_mask self.weighted_update = weighted_update self._replay_memory = ReplayMemory(mdp_info, initial_replay_size, max_replay_size, history_length, n_approximators, dtype) self._buffer = Buffer(history_length, dtype) self._n_updates = 0 self._episode_steps = 0 self._no_op_actions = None apprx_params_train = deepcopy(approximator_params) apprx_params_train['name'] = 'train' apprx_params_target = deepcopy(approximator_params) apprx_params_target['name'] = 'target' self.approximator = Regressor(approximator, **apprx_params_train) self.target_approximator = Regressor(approximator, **apprx_params_target) policy.set_q(self.approximator) self.target_approximator.model.set_weights( self.approximator.model.get_weights()) super(DQN, self).__init__(policy, mdp_info)
def test_multivariate_gaussian(): np.random.seed(88) n_dims = 5 n_outs = 3 random_matrix = np.random.rand(n_outs, n_outs) sigma = random_matrix.dot(random_matrix.T) approximator = Regressor(LinearApproximator, input_shape=(n_dims, ), output_shape=(n_outs, )) pi = GaussianPolicy(approximator, sigma) mu_weights = np.random.rand(pi.weights_size) pi.set_weights(mu_weights) x = np.random.randn(20, n_dims) for x_i in x: state = np.atleast_1d(x_i) action = pi.draw_action(state) exact_diff = pi.diff(state, action) numerical_diff = numerical_diff_policy(pi, state, action) assert np.allclose(exact_diff, numerical_diff)
def test_linear_approximator(): np.random.seed(88) noise = 1e-3 a = np.random.rand(1000, 3) k = np.random.rand(3, 2) b = a.dot(k) + np.random.randn(1000, 2) * noise approximator = Regressor(LinearApproximator, input_shape=(3, ), output_shape=(2, )) approximator.fit(a, b) khat = approximator.get_weights() deltaK = (khat - k.T.flatten()) assert np.linalg.norm(deltaK) < noise point = np.random.randn(3, ) derivative = approximator.diff(point) lp = len(point) for i in range(derivative.shape[1]): assert (derivative[i * lp:(i + 1) * lp, i] == point).all()
def univariate_gaussian(): print('Testing univariate gaussian policy...') sigma = 1e-3*np.eye(1) n_dims = 5 approximator_params = dict(input_dim=n_dims) approximator = Regressor(LinearApproximator, input_shape=(n_dims,), output_shape=(1,), params=approximator_params) pi = GaussianPolicy(approximator, sigma) mu_weights = np.random.rand(pi.weights_size) pi.set_weights(mu_weights) x = np.random.randn(20, n_dims) for x_i in x: state = np.atleast_1d(x_i) action = pi.draw_action(state) exact_diff = pi.diff(state, action) numerical_diff = numerical_diff_policy(pi, state, action) assert np.allclose(exact_diff, numerical_diff)
def build_low_level_ghavamzadeh(alg, params, mdp): # FeaturesL high = [150, 150, np.pi] low = [0, 0, -np.pi] n_tiles = [5, 5, 10] low = np.array(low, dtype=np.float) high = np.array(high, dtype=np.float) n_tilings = 3 tilingsL = Tiles.generate(n_tilings=n_tilings, n_tiles=n_tiles, low=low, high=high) featuresL = Features(tilings=tilingsL) mdp_info_agentL = MDPInfo(observation_space=spaces.Box( low=np.array([0, 0]), high=np.array([150, 150]), shape=(2, )), action_space=mdp.info.action_space, gamma=0.99, horizon=10000) input_shape = (featuresL.size, ) approximator = Regressor(LinearApproximator, input_shape=input_shape, output_shape=mdp.info.action_space.shape) std = np.array([3e-2]) pi = DiagonalGaussianPolicy(mu=approximator, std=std) agent = alg(pi, mdp_info_agentL, features=featuresL, **params) return agent
def experiment(alg, params, n_epochs, fit_per_run, ep_per_run): np.random.seed() # MDP mdp = LQR.generate(dimensions=1) approximator = Regressor(LinearApproximator, input_shape=mdp.info.observation_space.shape, output_shape=mdp.info.action_space.shape) policy = DeterministicPolicy(mu=approximator) mu = np.zeros(policy.weights_size) sigma = 1e-3 * np.eye(policy.weights_size) distribution = GaussianCholeskyDistribution(mu, sigma) # Agent agent = alg(distribution, policy, mdp.info, **params) # Train core = Core(agent, mdp) dataset_eval = core.evaluate(n_episodes=ep_per_run) print('distribution parameters: ', distribution.get_parameters()) J = compute_J(dataset_eval, gamma=mdp.info.gamma) print('J at start : ' + str(np.mean(J))) for i in range(n_epochs): core.learn(n_episodes=fit_per_run * ep_per_run, n_episodes_per_fit=ep_per_run) dataset_eval = core.evaluate(n_episodes=ep_per_run) print('distribution parameters: ', distribution.get_parameters()) J = compute_J(dataset_eval, gamma=mdp.info.gamma) print('J at iteration ' + str(i) + ': ' + str(np.mean(J)))
def test_deterministic_policy(): np.random.seed(88) n_dims = 5 approximator = Regressor(LinearApproximator, input_shape=(n_dims, ), output_shape=(2, )) pi = DeterministicPolicy(approximator) w_new = np.random.rand(pi.weights_size) w_old = pi.get_weights() pi.set_weights(w_new) assert np.array_equal(w_new, approximator.get_weights()) assert not np.array_equal(w_old, w_new) assert np.array_equal(w_new, pi.get_weights()) s_test_1 = np.random.randn(5) s_test_2 = np.random.randn(5) a_test = approximator.predict(s_test_1) assert pi.get_regressor() == approximator assert pi(s_test_1, a_test) == 1 assert pi(s_test_2, a_test) == 0 a_stored = np.array([-1.86941072, -0.1789696]) assert np.allclose(pi.draw_action(s_test_1), a_stored)
def test_pytorch_approximator(): np.random.seed(88) torch.manual_seed(88) noise = 1e-3**2 a = np.random.rand(1000, 4) k = np.random.rand(4, 2) b = np.sin(a).dot(k) + np.random.randn(1000, 2) * noise approximator = Regressor(PyTorchApproximator, input_shape=(4, ), output_shape=(2, ), network=ExampleNet, optimizer={ 'class': optim.Adam, 'params': {} }, loss=F.mse_loss, n_neurons=100, n_hidden=1, n_epochs=200, batch_size=100, quiet=True) approximator.fit(a, b) bhat = approximator.predict(a) error = np.linalg.norm(b - bhat, 'fro') / 1000 error_inf = np.max(np.abs(b - bhat)) print(b[:10]) print(bhat[:10]) print(error_inf) assert error < 2e-4 gradient = approximator.diff(a[0]) assert gradient.shape[1] == 2 old_weights = approximator.get_weights() approximator.set_weights(old_weights) new_weights = approximator.get_weights() assert np.array_equal(new_weights, old_weights) random_weights = np.random.randn(*old_weights.shape).astype(np.float32) approximator.set_weights(random_weights) random_weight_new = approximator.get_weights() assert np.array_equal(random_weights, random_weight_new) assert not np.any(np.equal(random_weights, old_weights)) bhat_random = approximator.predict(a) assert not np.array_equal(bhat, bhat_random)
def __init__(self, approximator, policy, mdp_info, batch_size, target_update_frequency, initial_replay_size, max_replay_size, fit_params=None, approximator_params=None, clip_reward=True, update_type='weighted', delta=0.1, store_prob=False, q_max=100, max_spread=None): self._fit_params = dict() if fit_params is None else fit_params self._batch_size = batch_size self._clip_reward = clip_reward self._target_update_frequency = target_update_frequency self.update_type = update_type self.delta = delta self.standard_bound = norm.ppf(1 - self.delta, loc=0, scale=1) self.store_prob = store_prob self.q_max = q_max self.max_spread = max_spread self._replay_memory = ReplayMemory(initial_replay_size, max_replay_size) self._n_updates = 0 self._epsilon = 1e-7 apprx_params_train = deepcopy(approximator_params) apprx_params_train['name'] = 'train' apprx_params_target = deepcopy(approximator_params) apprx_params_target['name'] = 'target' self.approximator = Regressor(approximator, **apprx_params_train) self.target_approximator = Regressor(approximator, **apprx_params_target) policy.set_q(self.approximator) self.target_approximator.model.set_weights( self.approximator.model.get_weights()) super(GaussianDQN, self).__init__(policy, mdp_info)
def build_high_level_agent(alg, params, mdp, mu, std): tilings = Tiles.generate(n_tilings=1, n_tiles=[10, 10], low=mdp.info.observation_space.low[:2], high=mdp.info.observation_space.high[:2]) features = Features(tilings=tilings) input_shape = (features.size, ) mu_approximator = Regressor(LinearApproximator, input_shape=input_shape, output_shape=(1, )) std_approximator = Regressor(LinearApproximator, input_shape=input_shape, output_shape=(1, )) w_mu = mu * np.ones(mu_approximator.weights_size) mu_approximator.set_weights(w_mu) w_std = std * np.ones(std_approximator.weights_size) mu_approximator.set_weights(w_std) pi = StateLogStdGaussianPolicy(mu=mu_approximator, log_std=std_approximator) obs_low = np.array( [mdp.info.observation_space.low[0], mdp.info.observation_space.low[1]]) obs_high = np.array([ mdp.info.observation_space.high[0], mdp.info.observation_space.high[1] ]) mdp_info_agent1 = MDPInfo(observation_space=spaces.Box(obs_low, obs_high, shape=(2, )), action_space=spaces.Box( mdp.info.observation_space.low[2], mdp.info.observation_space.high[2], shape=(1, )), gamma=1, horizon=10) agent = alg(policy=pi, mdp_info=mdp_info_agent1, features=features, **params) return agent
def experiment(alg, n_runs, n_iterations, ep_per_run, use_tensorflow): np.random.seed() # MDP mdp = ShipSteering() # Policy if use_tensorflow: tensor_list = gaussian_tensor.generate( [3, 3, 6, 2], [[0., 150.], [0., 150.], [-np.pi, np.pi], [-np.pi / 12, np.pi / 12]]) phi = Features(tensor_list=tensor_list, name='phi', input_dim=mdp.info.observation_space.shape[0]) else: basis = GaussianRBF.generate([3, 3, 6, 2], [[0., 150.], [0., 150.], [-np.pi, np.pi], [-np.pi / 12, np.pi / 12]]) phi = Features(basis_list=basis) input_shape = (phi.size, ) approximator_params = dict(input_dim=phi.size) approximator = Regressor(LinearApproximator, input_shape=input_shape, output_shape=mdp.info.action_space.shape, params=approximator_params) sigma = np.array([[.05]]) policy = MultivariateGaussianPolicy(mu=approximator, sigma=sigma) # Agent learning_rate = AdaptiveParameter(value=.01) algorithm_params = dict(learning_rate=learning_rate) fit_params = dict() agent_params = { 'algorithm_params': algorithm_params, 'fit_params': fit_params } agent = alg(policy, mdp.info, agent_params, phi) # Train core = Core(agent, mdp) dataset_eval = core.evaluate(n_episodes=ep_per_run) J = compute_J(dataset_eval, gamma=mdp.info.gamma) print('J at start : ' + str(np.mean(J))) for i in xrange(n_runs): core.learn(n_episodes=n_iterations * ep_per_run, n_episodes_per_fit=ep_per_run) dataset_eval = core.evaluate(n_episodes=ep_per_run) J = compute_J(dataset_eval, gamma=mdp.info.gamma) print('J at iteration ' + str(i) + ': ' + str(np.mean(J))) np.save('ship_steering.npy', dataset_eval)
def experiment(alg, params, experiment_params ,subdir, i): np.random.seed() # MDP mdp = ShipSteering(small=True, n_steps_action=3) high = [150, 150, np.pi] low = [0, 0, -np.pi] n_tiles = [5, 5, 6] low = np.array(low, dtype=np.float) high = np.array(high, dtype=np.float) n_tilings = 1 tilings = Tiles.generate(n_tilings=n_tilings, n_tiles=n_tiles, low=low, high=high) phi = Features(tilings=tilings) input_shape = (phi.size,) approximator_params = dict(input_dim=phi.size) approximator = Regressor(LinearApproximator, input_shape=input_shape, output_shape=mdp.info.action_space.shape, params=approximator_params) #sigma = np.array([[1e-4]]) std = np.array([3e-2]) policy = DiagonalGaussianPolicy(mu=approximator, std=std) #policy = GaussianPolicy(mu=approximator, sigma=sigma) # Agent agent = alg(policy, mdp.info, features=phi, **params) # Train parameter_dataset = CollectPolicyParameter(policy) core = Core(agent, mdp, callbacks=[parameter_dataset]) dataset_eval = list() dataset_eval_run = core.evaluate(n_episodes=ep_per_run) # print('distribution parameters: ', distribution.get_parameters()) J = compute_J(dataset_eval_run, gamma=mdp.info.gamma) dataset_eval += dataset_eval_run print('J at start : ' + str(np.mean(J))) for n in range(n_runs): print('ITERATION :', n) core.learn(n_episodes=n_iterations * ep_per_run, n_episodes_per_fit=ep_per_run) dataset_eval_run = core.evaluate(n_episodes=ep_per_run) J = compute_J(dataset_eval_run, gamma=mdp.info.gamma) print('J at iteration ' + str(n) + ': ' + str(np.mean(J))) dataset_eval += dataset_eval_run mk_dir_recursive('./' + subdir + str(i)) np.save(subdir+str(i)+'/dataset_eval_file', dataset_eval) np.save(subdir+str(i)+'/parameter_dataset_file', parameter_dataset)
def test_pytorch_approximator(): np.random.seed(1) torch.manual_seed(1) n_actions = 2 s = np.random.rand(1000, 4) a = np.random.randint(n_actions, size=(1000, 1)) q = np.random.rand(1000) approximator = Regressor(TorchApproximator, input_shape=(4,), output_shape=(2,), n_actions=n_actions, network=ExampleNet, optimizer={'class': optim.Adam, 'params': {}}, loss=F.mse_loss, batch_size=100, quiet=True) approximator.fit(s, a, q, n_epochs=20) x_s = np.random.rand(2, 4) x_a = np.random.randint(n_actions, size=(2, 1)) y = approximator.predict(x_s, x_a) y_test = np.array([0.37191153, 0.5920861]) assert np.allclose(y, y_test) y = approximator.predict(x_s) y_test = np.array([[0.47908658, 0.37191153], [0.5920861, 0.27575058]]) assert np.allclose(y, y_test) gradient = approximator.diff(x_s[0], x_a[0]) gradient_test = np.array([0., 0., 0., 0., 0.02627479, 0.76513696, 0.6672573, 0.35979462, 0., 1.]) assert np.allclose(gradient, gradient_test) gradient = approximator.diff(x_s[0]) gradient_test = np.array([[0.02627479, 0.], [0.76513696, 0.], [0.6672573, 0.], [0.35979462, 0.], [0., 0.02627479], [0., 0.76513696], [0., 0.6672573], [0., 0.35979462], [1, 0.], [0., 1.]]) assert np.allclose(gradient, gradient_test) old_weights = approximator.get_weights() approximator.set_weights(old_weights) new_weights = approximator.get_weights() assert np.array_equal(new_weights, old_weights) random_weights = np.random.randn(*old_weights.shape).astype(np.float32) approximator.set_weights(random_weights) random_weight_new = approximator.get_weights() assert np.array_equal(random_weights, random_weight_new) assert not np.any(np.equal(random_weights, old_weights))
def experiment(alg, params, subdir, exp_no): np.random.seed() # MDP mdp = ShipSteering(small=True, n_steps_action=3) high = [150, 150, np.pi] low = [0, 0, -np.pi] n_tiles = [5, 5, 6] low = np.array(low, dtype=np.float) high = np.array(high, dtype=np.float) n_tilings = 1 tilings = Tiles.generate(n_tilings=n_tilings, n_tiles=n_tiles, low=low, high=high) phi = Features(tilings=tilings) input_shape = (phi.size, ) approximator_params = dict(input_dim=input_shape) approximator = Regressor(LinearApproximator, input_shape=input_shape, output_shape=mdp.info.action_space.shape, params=approximator_params) policy = DeterministicPolicy(mu=approximator) mu = np.zeros(policy.weights_size) sigma = 4e-1 * np.ones(policy.weights_size) distribution = GaussianDiagonalDistribution(mu, sigma) # Agent agent = alg(distribution, policy, mdp.info, features=phi, **params) # Train dataset_eval = list() core = Core(agent, mdp) dataset_eval_run = core.evaluate(n_episodes=ep_per_run) #print('distribution parameters: ', distribution.get_parameters()) J = compute_J(dataset_eval_run, gamma=mdp.info.gamma) print('J at start : ' + str(np.mean(J))) dataset_eval += dataset_eval_run for n in range(n_runs): core.learn(n_episodes=n_iterations * ep_per_run, n_episodes_per_fit=ep_per_run) dataset_eval_run = core.evaluate(n_episodes=ep_per_run) J = compute_J(dataset_eval_run, gamma=mdp.info.gamma) print('J at iteration ' + str(n) + ': ' + str(np.mean(J))) dataset_eval += dataset_eval_run mk_dir_recursive('./' + subdir + str(exp_no)) np.save(subdir + str(exp_no) + '/dataset_eval_file', dataset_eval)
def experiment(alg, n_epochs, n_iterations, ep_per_run): np.random.seed() # MDP mdp = LQR.generate(dimensions=1) approximator_params = dict(input_dim=mdp.info.observation_space.shape) approximator = Regressor(LinearApproximator, input_shape=mdp.info.observation_space.shape, output_shape=mdp.info.action_space.shape, params=approximator_params) sigma = Regressor(LinearApproximator, input_shape=mdp.info.observation_space.shape, output_shape=mdp.info.action_space.shape, params=approximator_params) sigma_weights = 2 * np.ones(sigma.weights_size) sigma.set_weights(sigma_weights) policy = StateStdGaussianPolicy(approximator, sigma) # Agent learning_rate = AdaptiveParameter(value=.01) algorithm_params = dict(learning_rate=learning_rate) agent = alg(policy, mdp.info, **algorithm_params) # Train core = Core(agent, mdp) dataset_eval = core.evaluate(n_episodes=ep_per_run) print('policy parameters: ', policy.get_weights()) J = compute_J(dataset_eval, gamma=mdp.info.gamma) print('J at start : ' + str(np.mean(J))) for i in range(n_epochs): core.learn(n_episodes=n_iterations * ep_per_run, n_episodes_per_fit=ep_per_run) dataset_eval = core.evaluate(n_episodes=ep_per_run) print('policy parameters: ', policy.get_weights()) J = compute_J(dataset_eval, gamma=mdp.info.gamma) print('J at iteration ' + str(i) + ': ' + str(np.mean(J)))
def __init__(self, approximator, policy, mdp_info, batch_size, target_update_frequency, initial_replay_size, max_replay_size, fit_params=None, approximator_params=None, n_approximators=1, clip_reward=True, p_mask=2 / 3.): self._fit_params = dict() if fit_params is None else fit_params self._batch_size = batch_size self._n_approximators = n_approximators self._clip_reward = clip_reward self._target_update_frequency = target_update_frequency self._p_mask = p_mask self._replay_memory = ReplayMemory(initial_replay_size, max_replay_size) self._n_updates = 0 self._episode_steps = 0 apprx_params_train = deepcopy(approximator_params) apprx_params_train['name'] = 'train' apprx_params_target = deepcopy(approximator_params) apprx_params_target['name'] = 'target' self.approximator = Regressor(approximator, **apprx_params_train) self.target_approximator = Regressor(approximator, **apprx_params_target) policy.set_q(self.approximator) self.target_approximator.model.set_weights( self.approximator.model.get_weights()) super(BootstrappedDQN, self).__init__(policy, mdp_info)
def __init__(self, approximator, policy, mdp_info, params): alg_params = params['algorithm_params'] self._batch_size = alg_params.get('batch_size') self._n_approximators = alg_params.get('n_approximators', 1) self._clip_reward = alg_params.get('clip_reward', True) self._train_frequency = alg_params.get('train_frequency') self._target_update_frequency = alg_params.get( 'target_update_frequency') self._max_no_op_actions = alg_params.get('max_no_op_actions', 0) self._no_op_action_value = alg_params.get('no_op_action_value', 0) self._replay_memory = ReplayMemory( mdp_info, alg_params.get('initial_replay_size'), alg_params.get('max_replay_size'), alg_params.get('history_length', 1)) self._buffer = Buffer(size=alg_params.get('history_length', 1)) self._n_updates = 0 self._episode_steps = 0 self._no_op_actions = None apprx_params_train = deepcopy(params['approximator_params']) apprx_params_train['name'] = 'train' apprx_params_target = deepcopy(params['approximator_params']) apprx_params_target['name'] = 'target' self.approximator = Regressor(approximator, **apprx_params_train) self.target_approximator = Regressor(approximator, n_models=self._n_approximators, **apprx_params_target) policy.set_q(self.approximator) if self._n_approximators == 1: self.target_approximator.model.set_weights( self.approximator.model.get_weights()) else: for i in xrange(self._n_approximators): self.target_approximator.model[i].set_weights( self.approximator.model.get_weights()) super(DQN, self).__init__(policy, mdp_info, params)
def experiment(alg, params, n_epochs, n_iterations, ep_per_run): np.random.seed() # MDP mdp = ShipSteering() # Policy high = [150, 150, np.pi] low = [0, 0, -np.pi] n_tiles = [5, 5, 6] low = np.array(low, dtype=np.float) high = np.array(high, dtype=np.float) n_tilings = 1 tilings = Tiles.generate(n_tilings=n_tilings, n_tiles=n_tiles, low=low, high=high) phi = Features(tilings=tilings) input_shape = (phi.size, ) approximator = Regressor(LinearApproximator, input_shape=input_shape, output_shape=mdp.info.action_space.shape) policy = DeterministicPolicy(approximator) mu = np.zeros(policy.weights_size) sigma = 4e-1 * np.ones(policy.weights_size) distribution = GaussianDiagonalDistribution(mu, sigma) # Agent agent = alg(distribution, policy, mdp.info, features=phi, **params) # Train print(alg.__name__) core = Core(agent, mdp) dataset_eval = core.evaluate(n_episodes=ep_per_run) J = compute_J(dataset_eval, gamma=mdp.info.gamma) print('J at start : ' + str(np.mean(J))) for i in range(n_epochs): core.learn(n_episodes=n_iterations * ep_per_run, n_episodes_per_fit=ep_per_run) dataset_eval = core.evaluate(n_episodes=ep_per_run) J = compute_J(dataset_eval, gamma=mdp.info.gamma) print('J at iteration ' + str(i) + ': ' + str(np.mean(J)))
def build_high_level_agent(alg, params, mdp, mu, sigma): features = Features(basis_list=[PolynomialBasis()]) approximator = Regressor(LinearApproximator, input_shape=(features.size, ), output_shape=(2, )) approximator.set_weights(mu) pi1 = DiagonalGaussianPolicy(mu=approximator, std=sigma) lim = mdp.info.observation_space.high[0] mdp_info_agent = MDPInfo(observation_space=mdp.info.observation_space, action_space=spaces.Box(0, lim, (2, )), gamma=1.0, horizon=100) agent = alg(pi1, mdp_info_agent, features=features, **params) return agent
def build_approximator(mdp): high = [150, 150, np.pi] low = [0, 0, -np.pi] n_tiles = [5, 5, 8] low = np.array(low, dtype=np.float) high = np.array(high, dtype=np.float) n_tilings = 1 tilings = Tiles.generate(n_tilings=n_tilings, n_tiles=n_tiles, low=low, high=high, uniform=True) phi = Features(tilings=tilings) input_shape = (phi.size,) approximator = Regressor(LinearApproximator, input_shape=input_shape, output_shape=mdp.info.action_space.shape) return phi, approximator
def experiment(alg, n_runs, n_iterations, ep_per_run): np.random.seed() # MDP mdp = LQR.generate(dimensions=1) approximator_params = dict(input_dim=mdp.info.observation_space.shape) approximator = Regressor(LinearApproximator, input_shape=mdp.info.observation_space.shape, output_shape=mdp.info.action_space.shape, params=approximator_params) sigma = .1 * np.eye(1) policy = MultivariateGaussianPolicy(mu=approximator, sigma=sigma) # Agent learning_rate = AdaptiveParameter(value=.01) algorithm_params = dict(learning_rate=learning_rate) fit_params = dict() agent_params = { 'algorithm_params': algorithm_params, 'fit_params': fit_params } agent = alg(policy, mdp.info, agent_params) # Train core = Core(agent, mdp) dataset_eval = core.evaluate(n_episodes=ep_per_run) print 'policy parameters: ', policy.get_weights() J = compute_J(dataset_eval, gamma=mdp.info.gamma) print('J at start : ' + str(np.mean(J))) for i in xrange(n_runs): core.learn(n_episodes=n_iterations * ep_per_run, n_episodes_per_fit=ep_per_run) dataset_eval = core.evaluate(n_episodes=ep_per_run) print 'policy parameters: ', policy.get_weights() J = compute_J(dataset_eval, gamma=mdp.info.gamma) print('J at iteration ' + str(i) + ': ' + str(np.mean(J))) np.save('ship_steering.npy', dataset_eval)
def build_agent_low(alg, params, std, mdp): approximator = Regressor(LinearApproximator, input_shape=(3, ), output_shape=(1, )) n_weights = approximator.weights_size mu = np.zeros(n_weights) sigma = std * np.ones(n_weights) pi = DeterministicControlPolicy(approximator) dist = GaussianDiagonalDistribution(mu, sigma) # Agent Low mdp_info = MDPInfo( observation_space=spaces.Box( low=mdp.info.observation_space.low[1:], # FIXME FALSE high=mdp.info.observation_space.high[1:], # FIXME FALSE ), action_space=mdp.info.action_space, gamma=mdp.info.gamma, horizon=mdp.info.horizon) return alg(dist, pi, mdp_info, **params)
def build_agent_high(alg, params, std, mdp): # Features approximator1 = Regressor(LinearApproximator, input_shape=(1, ), output_shape=(1, )) # Policy H n_weights = approximator1.weights_size mu = np.zeros(n_weights) sigma = std * np.ones(n_weights) pi = DeterministicPolicy(approximator1) dist = GaussianDiagonalDistribution(mu, sigma) lim = np.pi / 2 low = mdp.info.observation_space.low[0:1] high = mdp.info.observation_space.high[0:1] mdp_info = MDPInfo(observation_space=spaces.Box(low, high), action_space=spaces.Box(-lim, lim, (1, )), gamma=mdp.info.gamma, horizon=mdp.info.horizon) return alg(dist, pi, mdp_info, **params)
def build_low_level_agent(alg, params, mdp, horizon, std): rho_max = np.linalg.norm(mdp.info.observation_space.high[:2] - mdp.info.observation_space.low[:2]) low = np.array([-np.pi, 0]) high = np.array([np.pi, rho_max]) basis = FourierBasis.generate(low, high, 10) features = Features(basis_list=basis) approximator = Regressor(LinearApproximator, input_shape=(features.size, ), output_shape=mdp.info.action_space.shape) pi = DiagonalGaussianPolicy(approximator, std) mdp_info_agent = MDPInfo(observation_space=spaces.Box(low, high), action_space=mdp.info.action_space, gamma=mdp.info.gamma, horizon=horizon) agent = alg(pi, mdp_info_agent, features=features, **params) return agent
def test_pytorch_approximator(): np.random.seed(88) torch.manual_seed(88) noise = 1e-3**2 a = np.random.rand(1000, 4) k = np.random.rand(4, 2) b = np.sin(a).dot(k) + np.random.randn(1000, 2) * noise approximator = Regressor(PyTorchApproximator, input_shape=(4, ), output_shape=(2, ), network=ExampleNet, optimizer={ 'class': optim.Adam, 'params': {} }, loss=F.mse_loss, n_neurons=100, n_hidden=1, n_epochs=200, batch_size=100, quiet=True) approximator.fit(a, b) bhat = approximator.predict(a) error = np.linalg.norm(b - bhat, 'fro') / 1000 error_inf = np.max(np.abs(b - bhat)) print(b[:10]) print(bhat[:10]) print(error_inf) assert error < 2e-4
def build_mid_level_agent(alg, params, mdp, mu, std): mu_approximator = Regressor(LinearApproximator, input_shape=(1, ), output_shape=(2, )) w_mu = mu * np.ones(mu_approximator.weights_size) mu_approximator.set_weights(w_mu) pi = DiagonalGaussianPolicy(mu=mu_approximator, std=std * np.ones(2)) lim = mdp.info.observation_space.high[0] basis = PolynomialBasis() features = BasisFeatures(basis=[basis]) mdp_info_agent1 = MDPInfo(observation_space=spaces.Box(0, 1, (1, )), action_space=spaces.Box(0, lim, (2, )), gamma=1, horizon=10) agent = alg(policy=pi, mdp_info=mdp_info_agent1, features=features, **params) return agent