def __call__(self, x):
network_out = self.network(x)
assert isinstance(
network_out,
dict) and 'mean' in network_out and 'logvar' in network_out
# Get mean and logvar for the action
mean = network_out['mean']
logvar = network_out['logvar']
# Obtain std: exp(0.5*log(std**2))
std = torch.exp(0.5 * logvar)
# Create indpendent normal distribution
action_dist = Normal(loc=mean, scale=std)
# Sample an action from the distribution
# We use PyTorch build-in reparameterized verion, rsample()
action = action_dist.rsample()
# Calculate log-probability of sampled action
action_logprob = action_dist.log_prob(action)
# Calculate entropy of the policy conditional on state
entropy = action_dist.entropy()
# Calculate perplexity of the policy, i.e. exp(entropy)
perplexity = action_dist.perplexity()
# Constraint action with lower/upper bounds
# TODO: where should we put before/after logprob ?
# https://discuss.pytorch.org/t/should-action-log-probability-computed-after-or-before-constraining-the-action/20976
# Note that it will be completely wrong if put constraint transformation
# before computing the log-probability. Because log-prob with transformed action is
# definitely a wrong value, it's equivalent to transformation of a Gaussian distribution
# and compute transformed samples with Gaussian density.
action = self.constraint_action(action)
# User-defined function to process any possible other output
processed_network_out = self.process_network_output(network_out)
# Dictionary of output
out = {}
out['action'] = action
out['action_logprob'] = action_logprob
out['entropy'] = entropy
out['perplexity'] = perplexity
# Augment with dictionary returned from processed network output
out = {**out, **processed_network_out}
return out
