def test_explained_variance_1d(self):
y = np.array([1, 2, 3, 4, 5, 0, 0, 0, 0, 0])
y_hat = np.array([2, 3, 4, 5, 6, 0, 0, 0, 0, 0])
valids = np.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
result = explained_variance_1d(y, y_hat, valids)
assert result == 1.0
result = explained_variance_1d(y, y_hat)
np.testing.assert_almost_equal(result, 0.95)
def optimize_policy(self, itr, samples_data):
"""Optimize policy.
Args:
itr (int): Iteration number.
samples_data (dict): Processed sample data.
See process_samples() for details.
"""
if self._fit_baseline == 'before':
self._fit_baseline_with_data(samples_data)
samples_data['baselines'] = self._get_baseline_prediction(
samples_data)
policy_opt_input_values = self._policy_opt_input_values(samples_data)
# Train policy network
logger.log('Computing loss before')
loss_before = self._optimizer.loss(policy_opt_input_values)
logger.log('Computing KL before')
policy_kl_before = self._f_policy_kl(*policy_opt_input_values)
logger.log('Optimizing')
self._optimizer.optimize(policy_opt_input_values)
logger.log('Computing KL after')
policy_kl = self._f_policy_kl(*policy_opt_input_values)
logger.log('Computing loss after')
loss_after = self._optimizer.loss(policy_opt_input_values)
tabular.record('{}/LossBefore'.format(self.policy.name), loss_before)
tabular.record('{}/LossAfter'.format(self.policy.name), loss_after)
tabular.record('{}/dLoss'.format(self.policy.name),
loss_before - loss_after)
tabular.record('{}/KLBefore'.format(self.policy.name),
policy_kl_before)
tabular.record('{}/KL'.format(self.policy.name), policy_kl)
pol_ent = self._f_policy_entropy(*policy_opt_input_values)
tabular.record('{}/Entropy'.format(self.policy.name), np.mean(pol_ent))
if self._fit_baseline == 'after':
self._fit_baseline_with_data(samples_data)
ev = np_tensor_utils.explained_variance_1d(samples_data['baselines'],
samples_data['returns'],
samples_data['valids'])
tabular.record('{}/ExplainedVariance'.format(self.baseline.name), ev)
def optimize_policy(self, samples_data):
"""Optimize policy.
Args:
samples_data (dict): Processed sample data.
See metarl.tf.paths_to_tensors() for details.
"""
policy_opt_input_values = self._policy_opt_input_values(samples_data)
logger.log('Computing loss before')
loss_before = self._optimizer.loss(policy_opt_input_values)
logger.log('Computing KL before')
policy_kl_before = self._f_policy_kl(*policy_opt_input_values)
logger.log('Optimizing')
self._optimizer.optimize(policy_opt_input_values)
logger.log('Computing KL after')
policy_kl = self._f_policy_kl(*policy_opt_input_values)
logger.log('Computing loss after')
loss_after = self._optimizer.loss(policy_opt_input_values)
tabular.record('{}/LossBefore'.format(self.policy.name), loss_before)
tabular.record('{}/LossAfter'.format(self.policy.name), loss_after)
tabular.record('{}/dLoss'.format(self.policy.name),
loss_before - loss_after)
tabular.record('{}/KLBefore'.format(self.policy.name),
policy_kl_before)
tabular.record('{}/KL'.format(self.policy.name), policy_kl)
pol_ent = self._f_policy_entropy(*policy_opt_input_values)
ent = np.sum(pol_ent) / np.sum(samples_data['valids'])
tabular.record('{}/Entropy'.format(self.policy.name), ent)
tabular.record('{}/Perplexity'.format(self.policy.name), np.exp(ent))
self._fit_baseline_with_data(samples_data)
ev = np_tensor_utils.explained_variance_1d(samples_data['baselines'],
samples_data['returns'],
samples_data['valids'])
tabular.record('{}/ExplainedVariance'.format(self._baseline.name), ev)
self._old_policy.model.parameters = self.policy.model.parameters