def learn(alg, alg_params): mdp = LQR.generate(dimensions=1) np.random.seed(1) torch.manual_seed(1) torch.cuda.manual_seed(1) approximator_params = dict(input_dim=mdp.info.observation_space.shape) approximator = Regressor(LinearApproximator, input_shape=mdp.info.observation_space.shape, output_shape=mdp.info.action_space.shape, params=approximator_params) sigma = Regressor(LinearApproximator, input_shape=mdp.info.observation_space.shape, output_shape=mdp.info.action_space.shape, params=approximator_params) sigma_weights = 2 * np.ones(sigma.weights_size) sigma.set_weights(sigma_weights) policy = StateStdGaussianPolicy(approximator, sigma) agent = alg(policy, mdp.info, **alg_params) core = Core(agent, mdp) core.learn(n_episodes=10, n_episodes_per_fit=5) return policy
def __init__(self, policy, mu, mdp_info, alpha_theta, alpha_omega, alpha_v, value_function_features=None, policy_features=None): self._mu = mu self._psi = value_function_features self._alpha_theta = alpha_theta self._alpha_omega = alpha_omega self._alpha_v = alpha_v if self._psi is not None: input_shape = (self._psi.size, ) else: input_shape = mdp_info.observation_space.shape self._V = Regressor(LinearApproximator, input_shape=input_shape, output_shape=(1, )) self._A = Regressor(LinearApproximator, input_shape=(self._mu.weights_size, ), output_shape=(1, )) super().__init__(policy, mdp_info, policy_features)
def __init__(self, actor_approximator, critic_approximator, policy_class, mdp_info, batch_size, initial_replay_size, max_replay_size, tau, actor_params, critic_params, policy_params, actor_fit_params=None, critic_fit_params=None): """ Constructor. Args: actor_approximator (object): the approximator to use for the actor; critic_approximator (object): the approximator to use for the critic; policy_class (Policy): class of the policy; batch_size (int): the number of samples in a batch; initial_replay_size (int): the number of samples to collect before starting the learning; max_replay_size (int): the maximum number of samples in the replay memory; tau (float): value of coefficient for soft updates; actor_params (dict): parameters of the actor approximator to build; critic_params (dict): parameters of the critic approximator to build; policy_params (dict): parameters of the policy to build; actor_fit_params (dict, None): parameters of the fitting algorithm of the actor approximator; critic_fit_params (dict, None): parameters of the fitting algorithm of the critic approximator; """ self._actor_fit_params = dict() if actor_fit_params is None else actor_fit_params self._critic_fit_params = dict() if critic_fit_params is None else critic_fit_params self._batch_size = batch_size self._tau = tau self._replay_memory = ReplayMemory(initial_replay_size, max_replay_size) target_critic_params = deepcopy(critic_params) self._critic_approximator = Regressor(critic_approximator, **critic_params) self._target_critic_approximator = Regressor(critic_approximator, **target_critic_params) if 'loss' not in actor_params: actor_params['loss'] = ActorLoss(self._critic_approximator) target_actor_params = deepcopy(actor_params) self._actor_approximator = Regressor(actor_approximator, **actor_params) self._target_actor_approximator = Regressor(actor_approximator, **target_actor_params) self._target_actor_approximator.model.set_weights( self._actor_approximator.model.get_weights()) self._target_critic_approximator.model.set_weights( self._critic_approximator.model.get_weights()) policy = policy_class(self._actor_approximator, **policy_params) super().__init__(policy, mdp_info)
def __init__(self, mdp_info, policy_class, policy_params, batch_size, initial_replay_size, max_replay_size, tau, critic_params, actor_params, actor_optimizer, policy_delay=1, critic_fit_params=None): """ Constructor. Args: policy_class (Policy): class of the policy; policy_params (dict): parameters of the policy to build; batch_size (int): the number of samples in a batch; initial_replay_size (int): the number of samples to collect before starting the learning; max_replay_size (int): the maximum number of samples in the replay memory; tau (float): value of coefficient for soft updates; actor_params (dict): parameters of the actor approximator to build; critic_params (dict): parameters of the critic approximator to build; actor_optimizer (dict): parameters to specify the actor optimizer algorithm; policy_delay (int, 1): the number of updates of the critic after which an actor update is implemented; critic_fit_params (dict, None): parameters of the fitting algorithm of the critic approximator; """ self._critic_fit_params = dict() if critic_fit_params is None else critic_fit_params self._batch_size = batch_size self._tau = tau self._policy_delay = policy_delay self._fit_count = 0 self._replay_memory = ReplayMemory(initial_replay_size, max_replay_size) target_critic_params = deepcopy(critic_params) self._critic_approximator = Regressor(TorchApproximator, **critic_params) self._target_critic_approximator = Regressor(TorchApproximator, **target_critic_params) target_actor_params = deepcopy(actor_params) self._actor_approximator = Regressor(TorchApproximator, **actor_params) self._target_actor_approximator = Regressor(TorchApproximator, **target_actor_params) self._init_target() policy = policy_class(self._actor_approximator, **policy_params) policy_parameters = self._actor_approximator.model.network.parameters() super().__init__(policy, mdp_info, actor_optimizer, policy_parameters)
def test_ornstein_uhlenbeck_policy(): np.random.seed(88) mu = Regressor(LinearApproximator, input_shape=(5, ), output_shape=(2, )) pi = OrnsteinUhlenbeckPolicy(mu, sigma=np.ones(1) * .2, theta=.15, dt=1e-2) w = np.random.randn(pi.weights_size) pi.set_weights(w) assert np.array_equal(pi.get_weights(), w) state = np.random.randn(5) action = pi.draw_action(state) action_test = np.array([-1.95896171, 1.91292747]) assert np.allclose(action, action_test) pi.reset() action = pi.draw_action(state) action_test = np.array([-1.94161061, 1.92233358]) assert np.allclose(action, action_test) try: pi(state, action) except NotImplementedError: pass else: assert False
def __init__(self, approximator, policy, mdp_info, n_iterations, fit_params=None, approximator_params=None, features=None, quiet=False): """ Constructor. Args: approximator (object): approximator used by the algorithm and the policy. n_iterations (int): number of iterations to perform for training; fit_params (dict, None): parameters of the fitting algorithm of the approximator; approximator_params (dict, None): parameters of the approximator to build; quiet (bool, False): whether to show the progress bar or not. """ self._n_iterations = n_iterations self._fit_params = dict() if fit_params is None else fit_params self._approximator_params = dict() if approximator_params is None else\ approximator_params self._quiet = quiet self.approximator = Regressor(approximator, **self._approximator_params) policy.set_q(self.approximator) super(BatchTD, self).__init__(policy, mdp_info, features)
def __init__(self, policy, mdp_info, alpha_theta, alpha_v, lambda_par=.9, value_function_features=None, policy_features=None): """ Constructor. Args: policy (ParametricPolicy): a differentiable stochastic policy; mdp_info: information about the MDP; alpha_theta (Parameter): learning rate for policy update; alpha_v (Parameter): learning rate for the value function; lambda_par (float, 0.9): trace decay parameter; value_function_features (Features, None): features used by the value function approximator; policy_features (Features, None): features used by the policy. """ self._psi = value_function_features self._alpha_theta = alpha_theta self._alpha_v = alpha_v self._lambda = lambda_par super().__init__(policy, mdp_info, policy_features) if self._psi is not None: input_shape = (self._psi.size,) else: input_shape = mdp_info.observation_space.shape self._V = Regressor(LinearApproximator, input_shape=input_shape, output_shape=(1,)) self._e_v = np.zeros(self._V.weights_size) self._e_theta = np.zeros(self.policy.weights_size)
def __init__(self, mdp_info, policy, critic_params, ent_coeff=0., max_kl=.001, lam=1., n_epochs_line_search=10, n_epochs_cg=10, cg_damping=1e-2, cg_residual_tol=1e-10, quiet=True, critic_fit_params=None): """ Constructor. Args: policy (TorchPolicy): torch policy to be learned by the algorithm critic_params (dict): parameters of the critic approximator to build; ent_coeff (float, 0): coefficient for the entropy penalty; max_kl (float, .001): maximum kl allowed for every policy update; lam float(float, 1.): lambda coefficient used by generalized advantage estimation; n_epochs_line_search (int, 10): maximum number of iterations of the line search algorithm; n_epochs_cg (int, 10): maximum number of iterations of the conjugate gradient algorithm; cg_damping (float, 1e-2): damping factor for the conjugate gradient algorithm; cg_residual_tol (float, 1e-10): conjugate gradient residual tolerance; quiet (bool, True): if true, the algorithm will print debug information; critic_fit_params (dict, None): parameters of the fitting algorithm of the critic approximator. """ self._critic_fit_params = dict( n_epochs=3) if critic_fit_params is None else critic_fit_params self._n_epochs_line_search = n_epochs_line_search self._n_epochs_cg = n_epochs_cg self._cg_damping = cg_damping self._cg_residual_tol = cg_residual_tol self._max_kl = max_kl self._ent_coeff = ent_coeff self._lambda = lam self._V = Regressor(TorchApproximator, **critic_params) self._iter = 1 self._quiet = quiet self._old_policy = None super().__init__(policy, mdp_info, None)
def __init__(self, mdp_info, policy, critic_params, ent_coeff=0., max_kl=.001, lam=1., n_epochs_line_search=10, n_epochs_cg=10, cg_damping=1e-2, cg_residual_tol=1e-10, quiet=True, critic_fit_params=None): """ Constructor. Args: """ self._critic_fit_params = dict(n_epochs=3) if critic_fit_params is None else critic_fit_params self._n_epochs_line_search = n_epochs_line_search self._n_epochs_cg = n_epochs_cg self._cg_damping = cg_damping self._cg_residual_tol = cg_residual_tol self._max_kl = max_kl self._ent_coeff = ent_coeff self._lambda = lam self._V = Regressor(TorchApproximator, **critic_params) self._iter = 1 self._quiet = quiet super().__init__(policy, mdp_info, None)
def __init__(self, approximator, policy, mdp_info, params, features): self.Q = Regressor(approximator, **params['approximator_params']) self.e = np.zeros(self.Q.weights_size) self._lambda = params['algorithm_params']['lambda'] super(SARSALambdaContinuous, self).__init__(self.Q, policy, mdp_info, params, features)
def __init__(self, approximator, policy, mdp_info, fit_params=None, approximator_params=None, features=None): """ Constructor. Args: approximator (object): approximator used by the algorithm and the policy. fit_params (dict, None): parameters of the fitting algorithm of the approximator; approximator_params (dict, None): parameters of the approximator to build; """ self._fit_params = dict() if fit_params is None else fit_params self._approximator_params = dict() if approximator_params is None else\ approximator_params self.approximator = Regressor(approximator, **self._approximator_params) policy.set_q(self.approximator) super().__init__(policy, mdp_info, features)
def __init__(self, policy, mdp_info, params, features): self.Q = Regressor(LinearApproximator, **params['approximator_params']) self.e = np.zeros(self.Q.weights_size) self._lambda = params['algorithm_params']['lambda'] self._q_old = None super(TrueOnlineSARSALambda, self).__init__(self.Q, policy, mdp_info, params, features)
def __init__(self, policy, mu, mdp_info, alpha_theta, alpha_omega, alpha_v, value_function_features=None, policy_features=None): """ Constructor. Args: policy (Policy): any exploration policy, possibly using the deterministic policy as mean regressor; mu (Regressor): regressor that describe the deterministic policy to be learned i.e., the deterministic mapping between state and action. alpha_theta (Parameter): learning rate for policy update; alpha_omega (Parameter): learning rate for the advantage function; alpha_v (Parameter): learning rate for the value function; value_function_features (Features, None): features used by the value function approximator; policy_features (Features, None): features used by the policy. """ self._mu = mu self._psi = value_function_features self._alpha_theta = alpha_theta self._alpha_omega = alpha_omega self._alpha_v = alpha_v if self._psi is not None: input_shape = (self._psi.size, ) else: input_shape = mdp_info.observation_space.shape self._V = Regressor(LinearApproximator, input_shape=input_shape, output_shape=(1, )) self._A = Regressor(LinearApproximator, input_shape=(self._mu.weights_size, ), output_shape=(1, )) super().__init__(policy, mdp_info, policy_features)
def experiment(n_epochs, n_episodes): np.random.seed() # MDP n_steps = 5000 mdp = InvertedPendulum(horizon=n_steps) # Agent n_tilings = 10 alpha_theta = Parameter(5e-3 / n_tilings) alpha_omega = Parameter(0.5 / n_tilings) alpha_v = Parameter(0.5 / n_tilings) tilings = Tiles.generate(n_tilings, [10, 10], mdp.info.observation_space.low, mdp.info.observation_space.high + 1e-3) phi = Features(tilings=tilings) input_shape = (phi.size,) mu = Regressor(LinearApproximator, input_shape=input_shape, output_shape=mdp.info.action_space.shape) sigma = 1e-1 * np.eye(1) policy = GaussianPolicy(mu, sigma) agent = COPDAC_Q(policy, mu, mdp.info, alpha_theta, alpha_omega, alpha_v, value_function_features=phi, policy_features=phi) # Train dataset_callback = CollectDataset() visualization_callback = Display(agent._V, mu, mdp.info.observation_space.low, mdp.info.observation_space.high, phi, phi) core = Core(agent, mdp, callbacks=[dataset_callback]) for i in range(n_epochs): core.learn(n_episodes=n_episodes, n_steps_per_fit=1, render=False) J = compute_J(dataset_callback.get(), gamma=1.0) dataset_callback.clean() visualization_callback() print('Mean Reward at iteration ' + str(i) + ': ' + str(np.sum(J) / n_steps / n_episodes)) print('Press a button to visualize the pendulum...') input() sigma = 1e-8 * np.eye(1) policy.set_sigma(sigma) core.evaluate(n_steps=n_steps, render=True)
def __init__(self, approximator, policy, mdp_info, params, features=None): """ Constructor. Args: approximator (object): approximator used by the algorithm and the policy. """ self._n_iterations = params['algorithm_params']['n_iterations'] self._quiet = params['algorithm_params'].get('quiet', False) self.approximator = Regressor(approximator, **params['approximator_params']) policy.set_q(self.approximator) super(BatchTD, self).__init__(policy, mdp_info, params, features)
def experiment(n_epochs, n_steps, n_eval_episodes): np.random.seed() # MDP mdp = InvertedPendulum() # Agent n_tilings = 10 alpha_theta = ExponentialDecayParameter(1, decay_exp=1.0) alpha_omega = ExponentialDecayParameter(1.5 / n_tilings, decay_exp=2 / 3) alpha_v = ExponentialDecayParameter(1 / n_tilings, decay_exp=2 / 3) tilings = Tiles.generate(n_tilings, [10, 10], mdp.info.observation_space.low, mdp.info.observation_space.high) phi = Features(tilings=tilings) input_shape = (phi.size, ) mu = Regressor(LinearApproximator, input_shape=input_shape, output_shape=mdp.info.action_space.shape) sigma = 1e-3 * np.eye(1) policy = GaussianPolicy(mu, sigma) agent = COPDAC_Q(policy, mu, mdp.info, alpha_theta, alpha_omega, alpha_v, value_function_features=phi, policy_features=phi) # Train core = Core(agent, mdp) dataset_eval = core.evaluate(n_episodes=n_eval_episodes) J = compute_J(dataset_eval, gamma=1.0) print('Total Reward per episode at start : ' + str(np.mean(J))) for i in range(n_epochs): core.learn(n_steps=n_steps, n_steps_per_fit=1) dataset_eval = core.evaluate(n_episodes=n_eval_episodes, render=False) J = compute_J(dataset_eval, gamma=1.0) print('Total Reward per episode at iteration ' + str(i) + ': ' + str(np.mean(J)))
def __init__(self, mdp_info, policy, critic_params, actor_optimizer, n_epochs_policy, batch_size, eps_ppo, lam, quiet=True, critic_fit_params=None): self._critic_fit_params = dict(n_epochs=10) if critic_fit_params is None else critic_fit_params self._n_epochs_policy = n_epochs_policy self._batch_size = batch_size self._eps_ppo = eps_ppo self._optimizer = actor_optimizer['class'](policy.parameters(), **actor_optimizer['params']) self._lambda = lam self._V = Regressor(TorchApproximator, **critic_params) self._quiet = quiet self._iter = 1 super().__init__(policy, mdp_info, None)
def experiment(n_epochs, ep_per_epoch_train, ep_per_epoch_eval, n_iterations): np.random.seed() # MDP mdp = PreyPredator() basis = PolynomialBasis.generate(1, mdp.info.observation_space.shape[0]) phi = Features(basis_list=basis[1:]) # Features approximator = Regressor(LinearApproximator, input_shape=(phi.size, ), output_shape=mdp.info.action_space.shape) sigma = 1e-2 * np.eye(mdp.info.action_space.shape[0]) policy = GaussianPolicy(approximator, sigma) lr = Parameter(1e-5) #agent = GPOMDP(policy, mdp.info, lr, phi) agent = KeyboardAgent() # Train core = Core(agent, mdp) dataset = core.evaluate(n_episodes=ep_per_epoch_eval, render=True) J = compute_J(dataset, gamma=mdp.info.gamma) print('Reward at start: ', np.mean(J)) for i in range(n_epochs): core.learn(n_episodes=ep_per_epoch_train, n_episodes_per_fit=ep_per_epoch_train // n_iterations, render=False) dataset = core.evaluate(n_episodes=ep_per_epoch_eval, render=True) J = compute_J(dataset, gamma=mdp.info.gamma) p = policy.get_weights() print('mu: ', p) print('Reward at iteration ', i, ': ', np.mean(J)) print('Press a button to visualize the segway...') input() core.evaluate(n_episodes=3, render=True)
def __init__(self, mdp_info, policy, critic_params, actor_optimizer, ent_coeff, max_grad_norm=None, critic_fit_params=None): """ Constructor. Args: policy (TorchPolicy): torch policy to be learned by the algorithm critic_params (dict): parameters of the critic approximator to build; actor_optimizer (dict): parameters to specify the actor optimizer algorithm; ent_coeff (float, 0): coefficient for the entropy penalty; max_grad_norm (float, None): maximum norm for gradient clipping. If None, no clipping will be performed, unless specified otherwise in actor_optimizer; critic_fit_params (dict, None): parameters of the fitting algorithm of the critic approximator. """ self._critic_fit_params = dict( ) if critic_fit_params is None else critic_fit_params self._entropy_coeff = ent_coeff self._V = Regressor(TorchApproximator, **critic_params) if 'clipping' not in actor_optimizer and max_grad_norm is not None: actor_optimizer = deepcopy(actor_optimizer) clipping_params = dict(max_norm=max_grad_norm, norm_type=2) actor_optimizer['clipping'] = dict( method=torch.nn.utils.clip_grad_norm_, params=clipping_params) super().__init__(policy, mdp_info, actor_optimizer, policy.parameters())
def experiment(alg, params, n_epochs, n_episodes, n_ep_per_fit): np.random.seed() # MDP mdp = Segway() # Policy approximator = Regressor(LinearApproximator, input_shape=mdp.info.observation_space.shape, output_shape=mdp.info.action_space.shape) n_weights = approximator.weights_size mu = np.zeros(n_weights) sigma = 2e-0 * np.ones(n_weights) policy = DeterministicPolicy(approximator) dist = GaussianDiagonalDistribution(mu, sigma) agent = alg(dist, policy, mdp.info, **params) # Train print(alg.__name__) dataset_callback = CollectDataset() core = Core(agent, mdp, callbacks=[dataset_callback]) for i in range(n_epochs): core.learn(n_episodes=n_episodes, n_episodes_per_fit=n_ep_per_fit, render=False) J = compute_J(dataset_callback.get(), gamma=mdp.info.gamma) dataset_callback.clean() p = dist.get_parameters() print('mu: ', p[:n_weights]) print('sigma: ', p[n_weights:]) print('Reward at iteration ' + str(i) + ': ' + str(np.mean(J))) print('Press a button to visualize the segway...') input() core.evaluate(n_episodes=3, render=True)
def test_copdac_q(): n_steps = 50 mdp = InvertedPendulum(horizon=n_steps) np.random.seed(1) torch.manual_seed(1) torch.cuda.manual_seed(1) # Agent n_tilings = 1 alpha_theta = Parameter(5e-3 / n_tilings) alpha_omega = Parameter(0.5 / n_tilings) alpha_v = Parameter(0.5 / n_tilings) tilings = Tiles.generate(n_tilings, [2, 2], mdp.info.observation_space.low, mdp.info.observation_space.high + 1e-3) phi = Features(tilings=tilings) input_shape = (phi.size,) mu = Regressor(LinearApproximator, input_shape=input_shape, output_shape=mdp.info.action_space.shape) sigma = 1e-1 * np.eye(1) policy = GaussianPolicy(mu, sigma) agent = COPDAC_Q(policy, mu, mdp.info, alpha_theta, alpha_omega, alpha_v, value_function_features=phi, policy_features=phi) # Train core = Core(agent, mdp) core.learn(n_episodes=2, n_episodes_per_fit=1) w = agent.policy.get_weights() w_test = np.array([0, -6.62180045e-7, 0, -4.23972882e-2]) assert np.allclose(w, w_test)
def __init__(self, mdp_info, policy, critic_params, actor_optimizer, n_epochs_policy, batch_size, eps_ppo, lam, quiet=True, critic_fit_params=None): """ Constructor. Args: policy (TorchPolicy): torch policy to be learned by the algorithm critic_params (dict): parameters of the critic approximator to build; actor_optimizer (dict): parameters to specify the actor optimizer algorithm; n_epochs_policy (int): number of policy updates for every dataset; batch_size (int): size of minibatches for every optimization step eps_ppo (float): value for probability ratio clipping; lam float(float, 1.): lambda coefficient used by generalized advantage estimation; quiet (bool, True): if true, the algorithm will print debug information; critic_fit_params (dict, None): parameters of the fitting algorithm of the critic approximator. """ self._critic_fit_params = dict(n_epochs=10) if critic_fit_params is None else critic_fit_params self._n_epochs_policy = n_epochs_policy self._batch_size = batch_size self._eps_ppo = eps_ppo self._optimizer = actor_optimizer['class'](policy.parameters(), **actor_optimizer['params']) self._lambda = lam self._V = Regressor(TorchApproximator, **critic_params) self._quiet = quiet self._iter = 1 super().__init__(policy, mdp_info, None)
def __init__(self, policy, mdp_info, learning_rate, lambda_coeff, features, approximator_params=None): """ Constructor. Args: lambda_coeff (float): eligibility trace coefficient. """ self._approximator_params = dict() if approximator_params is None else \ approximator_params self.Q = Regressor(LinearApproximator, **self._approximator_params) self.e = np.zeros(self.Q.weights_size) self._lambda = lambda_coeff self._q_old = None super().__init__(self.Q, policy, mdp_info, learning_rate, features)
def learn(alg, **alg_params): np.random.seed(1) torch.manual_seed(1) # MDP mdp = LQR.generate(dimensions=2) approximator = Regressor(LinearApproximator, input_shape=mdp.info.observation_space.shape, output_shape=mdp.info.action_space.shape) policy = DeterministicPolicy(mu=approximator) mu = np.zeros(policy.weights_size) sigma = 1e-3 * np.ones(policy.weights_size) distribution = GaussianDiagonalDistribution(mu, sigma) agent_test = alg(distribution, policy, mdp.info, **alg_params) core = Core(agent_test, mdp) core.learn(n_episodes=5, n_episodes_per_fit=5) return distribution
mdp = ShipSteering() high = [150, 150, np.pi] low = [0, 0, -np.pi] n_tiles = [5, 5, 6] low = np.array(low, dtype=np.float) high = np.array(high, dtype=np.float) n_tilings = 1 tilings = Tiles.generate(n_tilings=n_tilings, n_tiles=n_tiles, low=low, high=high) phi = Features(tilings=tilings) input_shape = (phi.size,) approximator = Regressor(LinearApproximator, input_shape=input_shape, output_shape=mdp.info.action_space.shape) policy = DeterministicPolicy(approximator) mu = np.zeros(policy.weights_size) sigma = 4e-1 * np.ones(policy.weights_size) distribution_test = GaussianDiagonalDistribution(mu, sigma) agent_test = RWR(distribution_test, policy, mdp.info, beta=1.) core = Core(agent_test, mdp) s = np.arange(10) a = np.arange(10) r = np.arange(10) ss = s + 5 ab = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1]) last = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1])
def learn(alg): n_steps = 50 mdp = InvertedPendulum(horizon=n_steps) np.random.seed(1) torch.manual_seed(1) torch.cuda.manual_seed(1) # Agent n_tilings = 2 alpha_r = Parameter(.0001) alpha_theta = Parameter(.001 / n_tilings) alpha_v = Parameter(.1 / n_tilings) tilings = Tiles.generate(n_tilings - 1, [1, 1], mdp.info.observation_space.low, mdp.info.observation_space.high + 1e-3) phi = Features(tilings=tilings) tilings_v = tilings + Tiles.generate( 1, [1, 1], mdp.info.observation_space.low, mdp.info.observation_space.high + 1e-3) psi = Features(tilings=tilings_v) input_shape = (phi.size, ) mu = Regressor(LinearApproximator, input_shape=input_shape, output_shape=mdp.info.action_space.shape) std = Regressor(LinearApproximator, input_shape=input_shape, output_shape=mdp.info.action_space.shape) std_0 = np.sqrt(1.) std.set_weights(np.log(std_0) / n_tilings * np.ones(std.weights_size)) policy = StateLogStdGaussianPolicy(mu, std) if alg is StochasticAC: agent = alg(policy, mdp.info, alpha_theta, alpha_v, lambda_par=.5, value_function_features=psi, policy_features=phi) elif alg is StochasticAC_AVG: agent = alg(policy, mdp.info, alpha_theta, alpha_v, alpha_r, lambda_par=.5, value_function_features=psi, policy_features=phi) core = Core(agent, mdp) core.learn(n_episodes=2, n_episodes_per_fit=1) return policy
def __init__(self, mdp_info, batch_size, initial_replay_size, max_replay_size, warmup_transitions, tau, lr_alpha, actor_mu_params, actor_sigma_params, actor_optimizer, critic_params, target_entropy=None, critic_fit_params=None): """ Constructor. Args: batch_size (int): the number of samples in a batch; initial_replay_size (int): the number of samples to collect before starting the learning; max_replay_size (int): the maximum number of samples in the replay memory; warmup_transitions (int): number of samples to accumulate in the replay memory to start the policy fitting; tau (float): value of coefficient for soft updates; lr_alpha (float): Learning rate for the entropy coefficient; actor_mu_params (dict): parameters of the actor mean approximator to build; actor_sigma_params (dict): parameters of the actor sigma approximator to build; actor_optimizer (dict): parameters to specify the actor optimizer algorithm; critic_params (dict): parameters of the critic approximator to build; target_entropy (float, None): target entropy for the policy, if None a default value is computed ; critic_fit_params (dict, None): parameters of the fitting algorithm of the critic approximator. """ self._critic_fit_params = dict() if critic_fit_params is None else critic_fit_params self._batch_size = batch_size self._warmup_transitions = warmup_transitions self._tau = tau if target_entropy is None: self._target_entropy = -np.prod(mdp_info.action_space.shape).astype(np.float32) else: self._target_entropy = target_entropy self._replay_memory = ReplayMemory(initial_replay_size, max_replay_size) if 'n_models' in critic_params.keys(): assert critic_params['n_models'] == 2 else: critic_params['n_models'] = 2 if 'prediction' in critic_params.keys(): assert critic_params['prediction'] == 'min' else: critic_params['prediction'] = 'min' target_critic_params = deepcopy(critic_params) self._critic_approximator = Regressor(TorchApproximator, **critic_params) self._target_critic_approximator = Regressor(TorchApproximator, **target_critic_params) self._log_alpha = torch.tensor(0., requires_grad=True, dtype=torch.float32) self._alpha_optim = optim.Adam([self._log_alpha], lr=lr_alpha) actor_mu_approximator = Regressor(TorchApproximator, **actor_mu_params) actor_sigma_approximator = Regressor(TorchApproximator, **actor_sigma_params) policy = SACPolicy(actor_mu_approximator, actor_sigma_approximator, mdp_info.action_space.low, mdp_info.action_space.high) self._init_target() policy_parameters = chain(actor_mu_approximator.model.network.parameters(), actor_sigma_approximator.model.network.parameters()) super().__init__(policy, mdp_info, actor_optimizer, policy_parameters)
import numpy as np from matplotlib import pyplot as plt from mushroom.approximators import Regressor from mushroom.approximators.parametric import LinearApproximator x = np.arange(10).reshape(-1, 1) intercept = 10 noise = np.random.randn(10, 1) * 1 y = 2 * x + intercept + noise phi = np.concatenate((np.ones(10).reshape(-1, 1), x), axis=1) regressor = Regressor(LinearApproximator, input_shape=(2, ), output_shape=(1, )) regressor.fit(phi, y) print('Weights: ' + str(regressor.get_weights())) print('Gradient: ' + str(regressor.diff(np.array([[5.]])))) plt.scatter(x, y) plt.plot(x, regressor.predict(phi)) plt.show()
import numpy as np from mushroom.algorithms.policy_search import * from mushroom.approximators import Regressor from mushroom.approximators.parametric import LinearApproximator from mushroom.core import Core from mushroom.environments.lqr import LQR from mushroom.policy.gaussian_policy import StateStdGaussianPolicy from mushroom.utils.parameters import AdaptiveParameter mdp = LQR.generate(dimensions=1) approximator_params = dict(input_dim=mdp.info.observation_space.shape) approximator = Regressor(LinearApproximator, input_shape=mdp.info.observation_space.shape, output_shape=mdp.info.action_space.shape, params=approximator_params) sigma = Regressor(LinearApproximator, input_shape=mdp.info.observation_space.shape, output_shape=mdp.info.action_space.shape, params=approximator_params) sigma_weights = 2 * np.ones(sigma.weights_size) sigma.set_weights(sigma_weights) policy = StateStdGaussianPolicy(approximator, sigma) # Agent learning_rate = AdaptiveParameter(value=.01) algorithm_params = dict(learning_rate=learning_rate)