class GaussianMLPBaselineWithModel(Baseline): """A value function using Gaussian MLP network.""" def __init__( self, env_spec, subsample_factor=1., num_seq_inputs=1, regressor_args=None, name='GaussianMLPBaselineWithModel', ): """ Constructor. :param env_spec: :param subsample_factor: :param num_seq_inputs: :param regressor_args: """ super().__init__(env_spec) if regressor_args is None: regressor_args = dict() self._regressor = GaussianMLPRegressorWithModel( input_shape=(env_spec.observation_space.flat_dim * num_seq_inputs, ), output_dim=1, name=name, **regressor_args) self.name = name @overrides def fit(self, paths): """Fit regressor based on paths.""" observations = np.concatenate([p['observations'] for p in paths]) returns = np.concatenate([p['returns'] for p in paths]) self._regressor.fit(observations, returns.reshape((-1, 1))) @overrides def predict(self, path): """Predict value based on paths.""" return self._regressor.predict(path['observations']).flatten() @overrides def get_param_values(self, **tags): """Get parameter values.""" return self._regressor.get_param_values(**tags) @overrides def set_param_values(self, flattened_params, **tags): """Set parameter values to val.""" self._regressor.set_param_values(flattened_params, **tags) @overrides def get_params_internal(self, **tags): """Get internal parameters.""" return self._regressor.get_params_internal(**tags)
def test_fit_smaller_subsample_factor(self): gmr = GaussianMLPRegressorWithModel( input_shape=(1, ), output_dim=1, subsample_factor=0.9) data = np.linspace(-np.pi, np.pi, 1000) obs = [{'observations': [[x]], 'returns': [np.sin(x)]} for x in data] observations = np.concatenate([p['observations'] for p in obs]) returns = np.concatenate([p['returns'] for p in obs]) for _ in range(150): gmr.fit(observations, returns.reshape((-1, 1))) paths = { 'observations': [[-np.pi], [-np.pi / 2], [-np.pi / 4], [0], [np.pi / 4], [np.pi / 2], [np.pi]] } prediction = gmr.predict(paths['observations']) expected = [[0], [-1], [-0.707], [0], [0.707], [1], [0]] assert np.allclose(prediction, expected, rtol=0, atol=0.1)
def test_fit_unnormalized(self): gmr = GaussianMLPRegressorWithModel( input_shape=(1, ), output_dim=1, subsample_factor=0.9, normalize_inputs=False, normalize_outputs=False) data = np.linspace(-np.pi, np.pi, 1000) obs = [{'observations': [[x]], 'returns': [np.sin(x)]} for x in data] observations = np.concatenate([p['observations'] for p in obs]) returns = np.concatenate([p['returns'] for p in obs]) for _ in range(150): gmr.fit(observations, returns.reshape((-1, 1))) paths = { 'observations': [[-np.pi], [-np.pi / 2], [-np.pi / 4], [0], [np.pi / 4], [np.pi / 2], [np.pi]] } prediction = gmr.predict(paths['observations']) expected = [[0], [-1], [-0.707], [0], [0.707], [1], [0]] assert np.allclose(prediction, expected, rtol=0, atol=0.1) x_mean = self.sess.run(gmr.model.networks['default'].x_mean) x_mean_expected = np.zeros_like(x_mean) x_std = self.sess.run(gmr.model.networks['default'].x_std) x_std_expected = np.ones_like(x_std) assert np.array_equal(x_mean, x_mean_expected) assert np.array_equal(x_std, x_std_expected) y_mean = self.sess.run(gmr.model.networks['default'].y_mean) y_mean_expected = np.zeros_like(y_mean) y_std = self.sess.run(gmr.model.networks['default'].y_std) y_std_expected = np.ones_like(y_std) assert np.allclose(y_mean, y_mean_expected) assert np.allclose(y_std, y_std_expected)
def test_fit_normalized(self): gmr = GaussianMLPRegressorWithModel(input_shape=(1, ), output_dim=1) data = np.linspace(-np.pi, np.pi, 1000) obs = [{'observations': [[x]], 'returns': [np.sin(x)]} for x in data] observations = np.concatenate([p['observations'] for p in obs]) returns = np.concatenate([p['returns'] for p in obs]) returns = returns.reshape((-1, 1)) for _ in range(150): gmr.fit(observations, returns) paths = { 'observations': [[-np.pi], [-np.pi / 2], [-np.pi / 4], [0], [np.pi / 4], [np.pi / 2], [np.pi]] } prediction = gmr.predict(paths['observations']) expected = [[0], [-1], [-0.707], [0], [0.707], [1], [0]] assert np.allclose(prediction, expected, rtol=0, atol=0.1) x_mean = self.sess.run(gmr.model.networks['default'].x_mean) x_mean_expected = np.mean(observations, axis=0, keepdims=True) x_std = self.sess.run(gmr.model.networks['default'].x_std) x_std_expected = np.std(observations, axis=0, keepdims=True) assert np.allclose(x_mean, x_mean_expected) assert np.allclose(x_std, x_std_expected) y_mean = self.sess.run(gmr.model.networks['default'].y_mean) y_mean_expected = np.mean(returns, axis=0, keepdims=True) y_std = self.sess.run(gmr.model.networks['default'].y_std) y_std_expected = np.std(returns, axis=0, keepdims=True) assert np.allclose(y_mean, y_mean_expected) assert np.allclose(y_std, y_std_expected)
class GaussianMLPBaselineWithModel(Baseline): """A value function using Gaussian MLP network.""" def __init__( self, env_spec, subsample_factor=1., num_seq_inputs=1, regressor_args=None, name='GaussianMLPBaselineWithModel', ): """ Gaussian MLP Baseline with Model. It fits the input data to a gaussian distribution estimated by a MLP. Args: env_spec (garage.envs.env_spec.EnvSpec): Environment specification. subsample_factor (float): The factor to subsample the data. By default it is 1.0, which means using all the data. num_seq_inputs (float): Number of sequence per input. By default it is 1.0, which means only one single sequence. regressor_args (dict): Arguments for regressor. """ super().__init__(env_spec) if regressor_args is None: regressor_args = dict() self._regressor = GaussianMLPRegressorWithModel( input_shape=(env_spec.observation_space.flat_dim * num_seq_inputs, ), output_dim=1, name=name, **regressor_args) self.name = name @overrides def fit(self, paths): """Fit regressor based on paths.""" observations = np.concatenate([p['observations'] for p in paths]) returns = np.concatenate([p['returns'] for p in paths]) self._regressor.fit(observations, returns.reshape((-1, 1))) @overrides def predict(self, path): """Predict value based on paths.""" return self._regressor.predict(path['observations']).flatten() @overrides def get_param_values(self, **tags): """Get parameter values.""" return self._regressor.get_param_values(**tags) @overrides def set_param_values(self, flattened_params, **tags): """Set parameter values to val.""" self._regressor.set_param_values(flattened_params, **tags) @overrides def get_params_internal(self, **tags): """Get internal parameters.""" return self._regressor.get_params_internal(**tags)