def test_multivariate_state_std_gaussian(): np.random.seed(88) n_dims = 5 n_outs = 3 mu_approximator = Regressor(LinearApproximator, input_shape=(n_dims, ), output_shape=(n_outs, )) std_approximator = Regressor(LinearApproximator, input_shape=(n_dims, ), output_shape=(n_outs, )) pi = StateStdGaussianPolicy(mu_approximator, std_approximator) weights = np.random.rand(pi.weights_size) + .1 pi.set_weights(weights) x = np.random.randn(20, n_dims) for x_i in x: state = np.atleast_1d(x_i) action = pi.draw_action(state) exact_diff = pi.diff(state, action) numerical_diff = numerical_diff_policy(pi, state, action) assert np.allclose(exact_diff, numerical_diff)
def experiment(n_epochs, n_iterations, ep_per_run, save_states_to_disk): np.random.seed() logger = Logger('plot_and_norm_example', results_dir=None) logger.strong_line() logger.info('Plotting and normalization example') # MDP mdp = LQR.generate(dimensions=2, max_pos=10., max_action=5., episodic=True) approximator = Regressor(LinearApproximator, input_shape=mdp.info.observation_space.shape, output_shape=mdp.info.action_space.shape) sigma = Regressor(LinearApproximator, input_shape=mdp.info.observation_space.shape, output_shape=mdp.info.action_space.shape) sigma_weights = 2 * np.ones(sigma.weights_size) sigma.set_weights(sigma_weights) policy = StateStdGaussianPolicy(approximator, sigma) # Agent optimizer = AdaptiveOptimizer(eps=.01) algorithm_params = dict(optimizer=optimizer) agent = REINFORCE(mdp.info, policy, **algorithm_params) # normalization callback prepro = MinMaxPreprocessor(mdp_info=mdp.info) # plotting callback plotter = PlotDataset(mdp.info, obs_normalized=True) # Train core = Core(agent, mdp, callback_step=plotter, preprocessors=[prepro]) # training loop for n in range(n_epochs): core.learn(n_episodes=n_iterations * ep_per_run, n_episodes_per_fit=ep_per_run) dataset = core.evaluate(n_episodes=ep_per_run, render=False) J = np.mean(compute_J(dataset, mdp.info.gamma)) logger.epoch_info(n + 1, J=J) if save_states_to_disk: # save normalization / plot states to disk path logger.info('Saving plotting and normalization data') os.makedirs("./logs/plot_and_norm", exist_ok=True) prepro.save("./logs/plot_and_norm/preprocessor.msh") plotter.save_state("./logs/plot_and_norm/plotting_state") # load states from disk path logger.info('Loading preprocessor and plotter') prerpo = MinMaxPreprocessor.load( "./logs/plot_and_norm/preprocessor.msh") plotter.load_state("./logs/plot_and_norm/plotting_state")
def _initialize_regressors(self, approximator, apprx_params_train, apprx_params_target): self.approximator = Regressor(approximator, **apprx_params_train) self.target_approximator = Regressor(approximator, n_models=self._n_approximators, **apprx_params_target) for i in range(len(self.target_approximator)): self.target_approximator[i].set_weights( self.approximator.get_weights() )
def _initialize_regressors(self, approximator, apprx_params_train, apprx_params_target): self.approximator = Regressor(approximator, n_models=self._n_approximators, prediction='min', **apprx_params_train) self.target_approximator = Regressor(approximator, n_models=self._n_approximators, prediction='min', **apprx_params_target) self._update_target()
def experiment(n_epochs, n_iterations, ep_per_run, save_states_to_disk): np.random.seed() # MDP mdp = LQR.generate(dimensions=2, max_pos=10., max_action=5., episodic=True) approximator = Regressor(LinearApproximator, input_shape=mdp.info.observation_space.shape, output_shape=mdp.info.action_space.shape) sigma = Regressor(LinearApproximator, input_shape=mdp.info.observation_space.shape, output_shape=mdp.info.action_space.shape) sigma_weights = 2 * np.ones(sigma.weights_size) sigma.set_weights(sigma_weights) policy = StateStdGaussianPolicy(approximator, sigma) # Agent learning_rate = AdaptiveParameter(value=.01) algorithm_params = dict(learning_rate=learning_rate) agent = REINFORCE(mdp.info, policy, **algorithm_params) # normalization callback prepro = MinMaxPreprocessor(mdp_info=mdp.info) # plotting callback plotter = PlotDataset(mdp.info, obs_normalized=True) # Train core = Core(agent, mdp, callback_step=plotter, preprocessors=[prepro]) # training loop for n in range(n_epochs): core.learn(n_episodes=n_iterations * ep_per_run, n_episodes_per_fit=ep_per_run) dataset = core.evaluate(n_episodes=ep_per_run, render=False) print('Epoch: ', n, ' J: ', np.mean(compute_J(dataset, mdp.info.gamma))) if save_states_to_disk: # save normalization / plot states to disk path os.makedirs("./temp/", exist_ok=True) prepro.save_state("./temp/normalization_state") plotter.save_state("./temp/plotting_state") # load states from disk path prepro.load_state("./temp/normalization_state") plotter.load_state("./temp/plotting_state")
def test_deterministic_policy(): np.random.seed(88) n_dims = 5 approximator = Regressor(LinearApproximator, input_shape=(n_dims,), output_shape=(2,)) pi = DeterministicPolicy(approximator) w_new = np.random.rand(pi.weights_size) w_old = pi.get_weights() pi.set_weights(w_new) assert np.array_equal(w_new, approximator.get_weights()) assert not np.array_equal(w_old, w_new) assert np.array_equal(w_new, pi.get_weights()) s_test_1 = np.random.randn(5) s_test_2 = np.random.randn(5) a_test = approximator.predict(s_test_1) assert pi.get_regressor() == approximator assert pi(s_test_1, a_test) == 1 assert pi(s_test_2, a_test) == 0 a_stored = np.array([-1.86941072, -0.1789696]) assert np.allclose(pi.draw_action(s_test_1), a_stored)
def test_torch_ensemble_logger(tmpdir): np.random.seed(1) torch.manual_seed(1) logger = Logger('ensemble_logger', results_dir=tmpdir, use_timestamp=True) approximator = Regressor(TorchApproximator, input_shape=(4,), output_shape=(2,), n_models=3, network=ExampleNet, optimizer={'class': optim.Adam, 'params': {}}, loss=F.mse_loss, batch_size=100, quiet=True) approximator.set_logger(logger) x = np.random.rand(1000, 4) y = np.random.rand(1000, 2) for i in range(50): approximator.fit(x, y) loss_file = np.load(logger.path / 'loss.npy') assert loss_file.shape == (50, 3) assert np.allclose(loss_file[0], np.array([0.29083753, 0.86829887, 1.0505845])) and \ np.allclose(loss_file[-1], np.array([0.09410495, 0.18786799, 0.15016919]))
def experiment(alg, params, n_epochs, fit_per_run, ep_per_run): np.random.seed() # MDP mdp = LQR.generate(dimensions=1) approximator = Regressor(LinearApproximator, input_shape=mdp.info.observation_space.shape, output_shape=mdp.info.action_space.shape) policy = DeterministicPolicy(mu=approximator) mu = np.zeros(policy.weights_size) sigma = 1e-3 * np.eye(policy.weights_size) distribution = GaussianCholeskyDistribution(mu, sigma) # Agent agent = alg(mdp.info, distribution, policy, **params) # Train core = Core(agent, mdp) dataset_eval = core.evaluate(n_episodes=ep_per_run) print('distribution parameters: ', distribution.get_parameters()) J = compute_J(dataset_eval, gamma=mdp.info.gamma) print('J at start : ' + str(np.mean(J))) for i in range(n_epochs): core.learn(n_episodes=fit_per_run * ep_per_run, n_episodes_per_fit=ep_per_run) dataset_eval = core.evaluate(n_episodes=ep_per_run) print('distribution parameters: ', distribution.get_parameters()) J = compute_J(dataset_eval, gamma=mdp.info.gamma) print('J at iteration ' + str(i) + ': ' + str(np.mean(J)))
def test_multivariate_gaussian(): np.random.seed(88) n_dims = 5 n_outs = 3 random_matrix = np.random.rand(n_outs, n_outs) sigma = random_matrix.dot(random_matrix.T) approximator = Regressor(LinearApproximator, input_shape=(n_dims, ), output_shape=(n_outs, )) pi = GaussianPolicy(approximator, sigma) mu_weights = np.random.rand(pi.weights_size) pi.set_weights(mu_weights) x = np.random.randn(20, n_dims) for x_i in x: state = np.atleast_1d(x_i) action = pi.draw_action(state) exact_diff = pi.diff(state, action) numerical_diff = numerical_diff_policy(pi, state, action) assert np.allclose(exact_diff, numerical_diff)
def experiment(alg, n_epochs, n_iterations, ep_per_run): np.random.seed() logger = Logger(alg.__name__, results_dir=None) logger.strong_line() logger.info('Experiment Algorithm: ' + alg.__name__) # MDP mdp = LQR.generate(dimensions=2, max_action=1., max_pos=1.) approximator = Regressor(LinearApproximator, input_shape=mdp.info.observation_space.shape, output_shape=mdp.info.action_space.shape) sigma = Regressor(LinearApproximator, input_shape=mdp.info.observation_space.shape, output_shape=mdp.info.action_space.shape) sigma_weights = 0.25 * np.ones(sigma.weights_size) sigma.set_weights(sigma_weights) policy = StateStdGaussianPolicy(approximator, sigma) # Agent optimizer = AdaptiveOptimizer(eps=1e-2) algorithm_params = dict(optimizer=optimizer) agent = alg(mdp.info, policy, **algorithm_params) # Train core = Core(agent, mdp) dataset_eval = core.evaluate(n_episodes=ep_per_run) J = compute_J(dataset_eval, gamma=mdp.info.gamma) logger.epoch_info(0, J=np.mean(J), policy_weights=policy.get_weights().tolist()) for i in trange(n_epochs, leave=False): core.learn(n_episodes=n_iterations * ep_per_run, n_episodes_per_fit=ep_per_run) dataset_eval = core.evaluate(n_episodes=ep_per_run) J = compute_J(dataset_eval, gamma=mdp.info.gamma) logger.epoch_info(i + 1, J=np.mean(J), policy_weights=policy.get_weights().tolist())
def test_pytorch_approximator(): np.random.seed(1) torch.manual_seed(1) n_actions = 2 s = np.random.rand(1000, 4) a = np.random.randint(n_actions, size=(1000, 1)) q = np.random.rand(1000) approximator = Regressor(TorchApproximator, input_shape=(4,), output_shape=(2,), n_actions=n_actions, network=ExampleNet, optimizer={'class': optim.Adam, 'params': {}}, loss=F.mse_loss, batch_size=100, quiet=True) approximator.fit(s, a, q, n_epochs=20) x_s = np.random.rand(2, 4) x_a = np.random.randint(n_actions, size=(2, 1)) y = approximator.predict(x_s, x_a) y_test = np.array([0.37191153, 0.5920861]) assert np.allclose(y, y_test) y = approximator.predict(x_s) y_test = np.array([[0.47908658, 0.37191153], [0.5920861, 0.27575058]]) assert np.allclose(y, y_test) gradient = approximator.diff(x_s[0], x_a[0]) gradient_test = np.array([0., 0., 0., 0., 0.02627479, 0.76513696, 0.6672573, 0.35979462, 0., 1.]) assert np.allclose(gradient, gradient_test) gradient = approximator.diff(x_s[0]) gradient_test = np.array([[0.02627479, 0.], [0.76513696, 0.], [0.6672573, 0.], [0.35979462, 0.], [0., 0.02627479], [0., 0.76513696], [0., 0.6672573], [0., 0.35979462], [1, 0.], [0., 1.]]) assert np.allclose(gradient, gradient_test) old_weights = approximator.get_weights() approximator.set_weights(old_weights) new_weights = approximator.get_weights() assert np.array_equal(new_weights, old_weights) random_weights = np.random.randn(*old_weights.shape).astype(np.float32) approximator.set_weights(random_weights) random_weight_new = approximator.get_weights() assert np.array_equal(random_weights, random_weight_new) assert not np.any(np.equal(random_weights, old_weights))
def experiment(alg, params, n_epochs, fit_per_epoch, ep_per_fit): np.random.seed() logger = Logger(alg.__name__, results_dir=None) logger.strong_line() logger.info('Experiment Algorithm: ' + alg.__name__) # MDP mdp = ShipSteering() # Policy high = [150, 150, np.pi] low = [0, 0, -np.pi] n_tiles = [5, 5, 6] low = np.array(low, dtype=np.float) high = np.array(high, dtype=np.float) n_tilings = 1 tilings = Tiles.generate(n_tilings=n_tilings, n_tiles=n_tiles, low=low, high=high) phi = Features(tilings=tilings) input_shape = (phi.size, ) approximator = Regressor(LinearApproximator, input_shape=input_shape, output_shape=mdp.info.action_space.shape) policy = DeterministicPolicy(approximator) mu = np.zeros(policy.weights_size) sigma = 4e-1 * np.ones(policy.weights_size) distribution = GaussianDiagonalDistribution(mu, sigma) # Agent agent = alg(mdp.info, distribution, policy, features=phi, **params) # Train core = Core(agent, mdp) dataset_eval = core.evaluate(n_episodes=ep_per_fit) J = compute_J(dataset_eval, gamma=mdp.info.gamma) logger.epoch_info(0, J=np.mean(J)) for i in range(n_epochs): core.learn(n_episodes=fit_per_epoch * ep_per_fit, n_episodes_per_fit=ep_per_fit) dataset_eval = core.evaluate(n_episodes=ep_per_fit) J = compute_J(dataset_eval, gamma=mdp.info.gamma) logger.epoch_info(i + 1, J=np.mean(J))
def experiment(alg, n_epochs, n_iterations, ep_per_run): np.random.seed() # MDP mdp = LQR.generate(dimensions=1) approximator = Regressor(LinearApproximator, input_shape=mdp.info.observation_space.shape, output_shape=mdp.info.action_space.shape) sigma = Regressor(LinearApproximator, input_shape=mdp.info.observation_space.shape, output_shape=mdp.info.action_space.shape) sigma_weights = 2 * np.ones(sigma.weights_size) sigma.set_weights(sigma_weights) policy = StateStdGaussianPolicy(approximator, sigma) # Agent learning_rate = AdaptiveParameter(value=.01) algorithm_params = dict(learning_rate=learning_rate) agent = alg(mdp.info, policy, **algorithm_params) # Train core = Core(agent, mdp) dataset_eval = core.evaluate(n_episodes=ep_per_run) print('policy parameters: ', policy.get_weights()) J = compute_J(dataset_eval, gamma=mdp.info.gamma) print('J at start : ' + str(np.mean(J))) for i in range(n_epochs): core.learn(n_episodes=n_iterations * ep_per_run, n_episodes_per_fit=ep_per_run) dataset_eval = core.evaluate(n_episodes=ep_per_run) print('policy parameters: ', policy.get_weights()) J = compute_J(dataset_eval, gamma=mdp.info.gamma) print('J at iteration ' + str(i) + ': ' + str(np.mean(J)))
def experiment(alg, params, n_epochs, n_iterations, ep_per_run): np.random.seed() # MDP mdp = ShipSteering() # Policy high = [150, 150, np.pi] low = [0, 0, -np.pi] n_tiles = [5, 5, 6] low = np.array(low, dtype=np.float) high = np.array(high, dtype=np.float) n_tilings = 1 tilings = Tiles.generate(n_tilings=n_tilings, n_tiles=n_tiles, low=low, high=high) phi = Features(tilings=tilings) input_shape = (phi.size, ) approximator = Regressor(LinearApproximator, input_shape=input_shape, output_shape=mdp.info.action_space.shape) policy = DeterministicPolicy(approximator) mu = np.zeros(policy.weights_size) sigma = 4e-1 * np.ones(policy.weights_size) distribution = GaussianDiagonalDistribution(mu, sigma) # Agent agent = alg(mdp.info, distribution, policy, features=phi, **params) # Train print(alg.__name__) core = Core(agent, mdp) dataset_eval = core.evaluate(n_episodes=ep_per_run) J = compute_J(dataset_eval, gamma=mdp.info.gamma) print('J at start : ' + str(np.mean(J))) for i in range(n_epochs): core.learn(n_episodes=n_iterations * ep_per_run, n_episodes_per_fit=ep_per_run) dataset_eval = core.evaluate(n_episodes=ep_per_run) J = compute_J(dataset_eval, gamma=mdp.info.gamma) print('J at iteration ' + str(i) + ': ' + str(np.mean(J)))
def experiment(alg, params, n_epochs, fit_per_epoch, ep_per_fit): np.random.seed() logger = Logger(alg.__name__, results_dir=None) logger.strong_line() logger.info('Experiment Algorithm: ' + alg.__name__) # MDP mdp = LQR.generate(dimensions=1) approximator = Regressor(LinearApproximator, input_shape=mdp.info.observation_space.shape, output_shape=mdp.info.action_space.shape) policy = DeterministicPolicy(mu=approximator) mu = np.zeros(policy.weights_size) sigma = 1e-3 * np.eye(policy.weights_size) distribution = GaussianCholeskyDistribution(mu, sigma) # Agent agent = alg(mdp.info, distribution, policy, **params) # Train core = Core(agent, mdp) dataset_eval = core.evaluate(n_episodes=ep_per_fit) J = compute_J(dataset_eval, gamma=mdp.info.gamma) logger.epoch_info(0, J=np.mean(J), distribution_parameters=distribution.get_parameters()) for i in trange(n_epochs, leave=False): core.learn(n_episodes=fit_per_epoch * ep_per_fit, n_episodes_per_fit=ep_per_fit) dataset_eval = core.evaluate(n_episodes=ep_per_fit) J = compute_J(dataset_eval, gamma=mdp.info.gamma) logger.epoch_info( i + 1, J=np.mean(J), distribution_parameters=distribution.get_parameters())
def test_cmac_approximator(): np.random.seed(1) # Generic regressor x = np.random.rand(1000, 2) k1 = np.random.rand(2) k2 = np.random.rand(2) y = np.array( [np.sin(x.dot(k1) * 2 * np.pi), np.sin(x.dot(k2) * 2 * np.pi)]).T tilings = Tiles.generate(10, [10, 10], np.zeros(2), np.ones(2)) approximator = Regressor(CMAC, tilings=tilings, input_shape=(2, ), output_shape=(2, )) approximator.fit(x, y) x = np.random.rand(2, 2) y_hat = approximator.predict(x) y_true = np.array( [np.sin(x.dot(k1) * 2 * np.pi), np.sin(x.dot(k2) * 2 * np.pi)]).T y_test = np.array([[-0.73581504, 0.90877225], [-0.95854488, -0.72429239]]) assert np.allclose(y_hat, y_test) point = np.random.rand(2) derivative = approximator.diff(point) assert np.array_equal(np.sum(derivative, axis=0), np.ones(2) * 10) assert len(derivative) == approximator.weights_size old_weights = approximator.get_weights() approximator.set_weights(old_weights) new_weights = approximator.get_weights() assert np.array_equal(new_weights, old_weights) random_weights = np.random.randn(*old_weights.shape).astype(np.float32) approximator.set_weights(random_weights) random_weight_new = approximator.get_weights() assert np.array_equal(random_weights, random_weight_new) assert not np.any(np.equal(random_weights, old_weights)) # Action regressor + Ensemble n_actions = 2 s = np.random.rand(1000, 3) a = np.random.randint(n_actions, size=(1000, 1)) q = np.random.rand(1000) tilings = Tiles.generate(10, [10, 10, 10], np.zeros(3), np.ones(3)) approximator = Regressor(CMAC, tilings=tilings, input_shape=(3, ), n_actions=n_actions, n_models=5) approximator.fit(s, a, q) np.random.seed(2) x_s = np.random.rand(2, 3) x_a = np.random.randint(n_actions, size=(2, 1)) y = approximator.predict(x_s, x_a, prediction='mean') y_test = np.array([[0.56235045, 0.25080909]]) assert np.allclose(y, y_test) y = approximator.predict(x_s, x_a, prediction='sum') y_test = np.array([2.81175226, 1.25404543]) assert np.allclose(y, y_test) y = approximator.predict(x_s, x_a, prediction='min') y_test = np.array([0.56235045, 0.25080909]) assert np.allclose(y, y_test) y = approximator.predict(x_s) y_test = np.array([[0.10367145, 0.56235045], [0.05575822, 0.25080909]]) assert np.allclose(y, y_test)
def test_linear_approximator(): np.random.seed(1) # Generic regressor a = np.random.rand(1000, 3) k = np.random.rand(3, 2) b = a.dot(k) + np.random.randn(1000, 2) approximator = Regressor(LinearApproximator, input_shape=(3, ), output_shape=(2, )) approximator.fit(a, b) x = np.random.rand(2, 3) y = approximator.predict(x) y_test = np.array([[0.57638247, 0.1573216], [0.11388247, 0.24123678]]) assert np.allclose(y, y_test) point = np.random.randn(3, ) derivative = approximator.diff(point) lp = len(point) for i in range(derivative.shape[1]): assert (derivative[i * lp:(i + 1) * lp, i] == point).all() old_weights = approximator.get_weights() approximator.set_weights(old_weights) new_weights = approximator.get_weights() assert np.array_equal(new_weights, old_weights) random_weights = np.random.randn(*old_weights.shape).astype(np.float32) approximator.set_weights(random_weights) random_weight_new = approximator.get_weights() assert np.array_equal(random_weights, random_weight_new) assert not np.any(np.equal(random_weights, old_weights)) # Action regressor + Ensemble n_actions = 2 s = np.random.rand(1000, 3) a = np.random.randint(n_actions, size=(1000, 1)) q = np.random.rand(1000) approximator = Regressor(LinearApproximator, input_shape=(3, ), n_actions=n_actions, n_models=5) approximator.fit(s, a, q) x_s = np.random.rand(2, 3) x_a = np.random.randint(n_actions, size=(2, 1)) y = approximator.predict(x_s, x_a, prediction='mean') y_test = np.array([0.49225698, 0.69660881]) assert np.allclose(y, y_test) y = approximator.predict(x_s, x_a, prediction='sum') y_test = np.array([2.46128492, 3.48304404]) assert np.allclose(y, y_test) y = approximator.predict(x_s, x_a, prediction='min') y_test = np.array([[0.49225698, 0.69660881]]) assert np.allclose(y, y_test) y = approximator.predict(x_s) y_test = np.array([[0.49225698, 0.44154141], [0.69660881, 0.69060195]]) assert np.allclose(y, y_test) approximator = Regressor(LinearApproximator, input_shape=(3, ), n_actions=n_actions) approximator.fit(s, a, q) gradient = approximator.diff(x_s[0], x_a[0]) gradient_test = np.array([0.88471362, 0.11666548, 0.45466254, 0., 0., 0.]) assert np.allclose(gradient, gradient_test)
def __init__(self, mdp_info, policy, approximator, approximator_params, batch_size, target_update_frequency, replay_memory=None, initial_replay_size=500, max_replay_size=5000, fit_params=None, n_approximators=1, clip_reward=True): """ Constructor. Args: approximator (object): the approximator to use to fit the Q-function; approximator_params (dict): parameters of the approximator to build; batch_size (int): the number of samples in a batch; target_update_frequency (int): the number of samples collected between each update of the target network; replay_memory ([ReplayMemory, PrioritizedReplayMemory], None): the object of the replay memory to use; if None, a default replay memory is created; initial_replay_size (int): the number of samples to collect before starting the learning; max_replay_size (int): the maximum number of samples in the replay memory; fit_params (dict, None): parameters of the fitting algorithm of the approximator; n_approximators (int, 1): the number of approximator to use in ``AveragedDQN``; clip_reward (bool, True): whether to clip the reward or not. """ self._fit_params = dict() if fit_params is None else fit_params self._batch_size = batch_size self._n_approximators = n_approximators self._clip_reward = clip_reward self._target_update_frequency = target_update_frequency if replay_memory is not None: self._replay_memory = replay_memory if isinstance(replay_memory, PrioritizedReplayMemory): self._fit = self._fit_prioritized else: self._fit = self._fit_standard else: self._replay_memory = ReplayMemory(initial_replay_size, max_replay_size) self._fit = self._fit_standard self._n_updates = 0 apprx_params_train = deepcopy(approximator_params) apprx_params_target = deepcopy(approximator_params) self.approximator = Regressor(approximator, **apprx_params_train) self.target_approximator = Regressor(approximator, n_models=self._n_approximators, **apprx_params_target) policy.set_q(self.approximator) if self._n_approximators == 1: self.target_approximator.set_weights( self.approximator.get_weights()) else: for i in range(self._n_approximators): self.target_approximator[i].set_weights( self.approximator.get_weights()) self._add_save_attr(_fit_params='pickle', _batch_size='primitive', _n_approximators='primitive', _clip_reward='primitive', _target_update_frequency='primitive', _replay_memory='mushroom', _n_updates='primitive', approximator='mushroom', target_approximator='mushroom') super().__init__(mdp_info, policy)
def test_cmac_approximator(): np.random.seed(1) # Generic regressor x = np.random.rand(1000, 2) k1 = np.random.rand(2) k2 = np.random.rand(2) y = np.array( [np.sin(x.dot(k1) * 2 * np.pi), np.sin(x.dot(k2) * 2 * np.pi)]).T tilings = Tiles.generate(10, [10, 10], np.zeros(2), np.ones(2)) approximator = Regressor(CMAC, tilings=tilings, input_shape=(2, ), output_shape=(2, )) approximator.fit(x, y) x = np.random.rand(2, 2) y_hat = approximator.predict(x) y_true = np.array( [np.sin(x.dot(k1) * 2 * np.pi), np.sin(x.dot(k2) * 2 * np.pi)]).T y_test = np.array([[-0.73787754, 0.90673493], [-0.94972964, -0.72380013]]) assert np.allclose(y_hat, y_test) point = np.random.rand(2) derivative = approximator.diff(point) assert np.array_equal(np.sum(derivative, axis=0), np.ones(2) * 10) assert len(derivative) == approximator.weights_size old_weights = approximator.get_weights() approximator.set_weights(old_weights) new_weights = approximator.get_weights() assert np.array_equal(new_weights, old_weights) random_weights = np.random.randn(*old_weights.shape).astype(np.float32) approximator.set_weights(random_weights) random_weight_new = approximator.get_weights() assert np.array_equal(random_weights, random_weight_new) assert not np.any(np.equal(random_weights, old_weights)) # Action regressor + Ensemble n_actions = 2 s = np.random.rand(1000, 3) a = np.random.randint(n_actions, size=(1000, 1)) q = np.random.rand(1000) tilings = Tiles.generate(10, [10, 10, 10], np.zeros(3), np.ones(3)) approximator = Regressor(CMAC, tilings=tilings, input_shape=(3, ), n_actions=n_actions, n_models=5) approximator.fit(s, a, q) x_s = np.random.rand(2, 3) x_a = np.random.randint(n_actions, size=(2, 1)) y = approximator.predict(x_s, x_a, prediction='mean') y_test = np.array([[0.10921918, 0.09923379]]) assert np.allclose(y, y_test) y = approximator.predict(x_s, x_a, prediction='sum') y_test = np.array([0.54609592, 0.49616895]) assert np.allclose(y, y_test) y = approximator.predict(x_s, x_a, prediction='min') y_test = np.array([[0.10921918, 0.09923379]]) assert np.allclose(y, y_test) y = approximator.predict(x_s) y_test = np.array([[0.07606651, 0.10921918], [0.40698114, 0.09923379]]) assert np.allclose(y, y_test)
def _initialize_regressors(self, approximator, apprx_params_train, apprx_params_target): self.approximator = Regressor(approximator, **apprx_params_train) self.target_approximator = Regressor(approximator, **apprx_params_target) self._update_target()