class Beta(Distribution):
r"""
Beta distribution parameterized by `concentration1` and `concentration0`.
Example::
>>> m = Beta(torch.Tensor([0.5]), torch.Tensor([0.5]))
>>> m.sample() # Beta distributed with concentration concentration1 and concentration0
0.1046
[torch.FloatTensor of size 1]
Args:
concentration1 (float or Tensor or Variable): 1st concentration parameter of the distribution
(often referred to as alpha)
concentration0 (float or Tensor or Variable): 2nd concentration parameter of the distribution
(often referred to as beta)
"""
params = {'concentration1': constraints.positive, 'concentration0': constraints.positive}
support = constraints.unit_interval
has_rsample = True
def __init__(self, concentration1, concentration0):
if isinstance(concentration1, Number) and isinstance(concentration0, Number):
concentration1_concentration0 = torch.Tensor([concentration1, concentration0])
else:
concentration1, concentration0 = broadcast_all(concentration1, concentration0)
concentration1_concentration0 = torch.stack([concentration1, concentration0], -1)
self._dirichlet = Dirichlet(concentration1_concentration0)
super(Beta, self).__init__(self._dirichlet._batch_shape)
def rsample(self, sample_shape=()):
value = self._dirichlet.rsample(sample_shape).select(-1, 0)
if isinstance(value, Number):
value = self._dirichlet.concentration.new([value])
return value
def log_prob(self, value):
self._validate_log_prob_arg(value)
heads_tails = torch.stack([value, 1.0 - value], -1)
return self._dirichlet.log_prob(heads_tails)
def entropy(self):
return self._dirichlet.entropy()
@property
def concentration1(self):
result = self._dirichlet.concentration[..., 0]
if isinstance(result, Number):
return torch.Tensor([result])
else:
return result
@property
def concentration0(self):
result = self._dirichlet.concentration[..., 1]
if isinstance(result, Number):
return torch.Tensor([result])
else:
return result
class Beta(Distribution):
r"""
Creates a Beta distribution parameterized by concentration `alpha` and `beta`.
Example::
>>> m = Beta(torch.Tensor([0.5]), torch.Tensor([0.5]))
>>> m.sample() # Beta distributed with concentration alpha and beta
0.1046
[torch.FloatTensor of size 1]
Args:
alpha (float or Tensor or Variable): 1st concentration parameter of the distribution
beta (float or Tensor or Variable): 2nd concentration parameter of the distribution
"""
params = {'alpha': constraints.positive, 'beta': constraints.positive}
support = constraints.unit_interval
has_rsample = True
def __init__(self, alpha, beta):
if isinstance(alpha, Number) and isinstance(beta, Number):
alpha_beta = torch.Tensor([alpha, beta])
else:
alpha, beta = broadcast_all(alpha, beta)
alpha_beta = torch.stack([alpha, beta], -1)
self._dirichlet = Dirichlet(alpha_beta)
super(Beta, self).__init__(self._dirichlet._batch_shape)
def rsample(self, sample_shape=()):
value = self._dirichlet.rsample(sample_shape).select(-1, 0)
if isinstance(value, Number):
value = self._dirichlet.alpha.new([value])
return value
def log_prob(self, value):
self._validate_log_prob_arg(value)
heads_tails = torch.stack([value, 1.0 - value], -1)
return self._dirichlet.log_prob(heads_tails)
def entropy(self):
return self._dirichlet.entropy()
@property
def alpha(self):
result = self._dirichlet.alpha[..., 0]
if isinstance(result, Number):
return torch.Tensor([result])
else:
return result
@property
def beta(self):
result = self._dirichlet.alpha[..., 1]
if isinstance(result, Number):
return torch.Tensor([result])
else:
return result
def log_joint_pdf(self, samples, log_f):
"""
Returns (the log of) p(piece | cluster, transposition); assuming uniform prior over clusters and transpositions,
this is proportional to the joint p(piece, cluster, transposition)
:param samples: array of shape (n_pieces, n_samples, n_pitches, n_clusters, n_transpositions)
:param log_f: array of shape (n_pieces, n_samples, n_pitches, n_clusters, n_transpositions)
:return: array of shape (...)
"""
# construct Dirichlet distributions (move pitch dimension last
dir = Dirichlet(torch.einsum('abcde->abdec', log_f.exp()))
# get point-wise probabilities and multiply up (log-sum) samples for each piece
probs = dir.log_prob(torch.einsum('abcde->abdec', samples))
return probs.sum(dim=1)
def loss_function(targets,
outputs,
mu,
logvar,
alphas,
topics,
bow=None,
joint=False):
"""
Inputs:
targets: target tokens
outputs: predicted tokens
mu: latent mean
logvar: log of the latent variance
alphas: parameters of the dirichlet prior p(w|z) given latent code
topics: actual distribution of topics q(w|x,z) i.e. posterior given x
Outputs:
ce_loss: cross entropy loss of the tokens
kld: D(q(z|x)||p(z))
kld_tpc: D(q(w|x,z)||p(w|z))
"""
ce_loss = F.cross_entropy(outputs.view(
outputs.size(0) * outputs.size(1), outputs.size(2)),
targets.view(-1),
size_average=False,
ignore_index=PAD_ID)
if bow is None:
bow_loss = torch.tensor(0., device=outputs.device)
else:
bow = bow.unsqueeze(1).repeat(1, outputs.size(1), 1).contiguous()
bow_loss = F.cross_entropy(bow.view(
bow.size(0) * bow.size(1), bow.size(2)),
targets.view(-1),
size_average=False,
ignore_index=PAD_ID)
if type(mu) == torch.Tensor:
kld = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
else:
kld = -0.5 * torch.sum(1 + logvar[1] - logvar[0] - (
(mu[1] - mu[0]).pow(2) + logvar[1].exp()) / logvar[0].exp())
prior = Dirichlet(alphas)
if joint:
loss_tpc = -torch.sum(prior.log_prob(topics))
else:
alphas2 = topics * topics.size(1)
posterior = Dirichlet(alphas2)
loss_tpc = kl_divergence(posterior, prior).sum()
return ce_loss, kld, loss_tpc, bow_loss
def test_dirichlet_logpdf():
alpha = torch.tensor([0.5, 0.6, 1.2])
ps = torch.tensor([0.2, 0.3, 0.5])
log_pdf = dirichlet_logpdf(ps, alpha)
# pytorch implementation
dist = Dirichlet(concentration=alpha)
log_prob = dist.log_prob(ps)
print(log_pdf)
print(log_prob)
assert log_pdf == log_prob
class Beta(Distribution):
r"""
Creates a Beta distribution parameterized by concentration `alpha` and `beta`.
Example::
>>> m = Beta(torch.Tensor([0.5]), torch.Tensor([0.5]))
>>> m.sample() # Beta distributed with concentrarion alpha
0.1046
[torch.FloatTensor of size 2]
Args:
alpha (Tensor or Variable): concentration parameter of the distribution
"""
params = {'alpha': constraints.positive, 'beta': constraints.positive}
support = constraints.unit_interval
has_rsample = True
def __init__(self, alpha, beta):
if isinstance(alpha, Number) and isinstance(beta, Number):
alpha_beta = torch.Tensor([alpha, beta])
else:
alpha, beta = broadcast_all(alpha, beta)
alpha_beta = torch.stack([alpha, beta], -1)
self._dirichlet = Dirichlet(alpha_beta)
super(Beta, self).__init__(self._dirichlet._batch_shape)
def rsample(self, sample_shape=()):
value = self._dirichlet.rsample(sample_shape).select(-1, 0)
if isinstance(value, Number):
value = self._dirichlet.alpha.new([value])
return value
def log_prob(self, value):
self._validate_log_prob_arg(value)
heads_tails = torch.stack([value, 1.0 - value], -1)
return self._dirichlet.log_prob(heads_tails)
def entropy(self):
return self._dirichlet.entropy()
class Beta(Distribution):
r"""
Creates a Beta distribution parameterized by concentration `alpha` and `beta`.
Example::
>>> m = Beta(torch.Tensor([0.5]), torch.Tensor([0.5]))
>>> m.sample() # Beta distributed with concentrarion alpha
0.1046
[torch.FloatTensor of size 2]
Args:
alpha (Tensor or Variable): concentration parameter of the distribution
"""
has_rsample = True
def __init__(self, alpha, beta):
if isinstance(alpha, Number) and isinstance(beta, Number):
alpha_beta = torch.Tensor([alpha, beta])
else:
alpha, beta = broadcast_all(alpha, beta)
alpha_beta = torch.stack([alpha, beta], -1)
self._dirichlet = Dirichlet(alpha_beta)
super(Beta, self).__init__(self._dirichlet._batch_shape)
def rsample(self, sample_shape=()):
value = self._dirichlet.rsample(sample_shape).select(-1, 0)
if isinstance(value, Number):
value = self._dirichlet.alpha.new([value])
return value
def log_prob(self, value):
self._validate_log_prob_arg(value)
heads_tails = torch.stack([value, 1.0 - value], -1)
return self._dirichlet.log_prob(heads_tails)
def entropy(self):
return self._dirichlet.entropy()
class Beta(ExponentialFamily):
r"""
Beta distribution parameterized by `concentration1` and `concentration0`.
Example::
>>> m = Beta(torch.tensor([0.5]), torch.tensor([0.5]))
>>> m.sample() # Beta distributed with concentration concentration1 and concentration0
0.1046
[torch.FloatTensor of size 1]
Args:
concentration1 (float or Tensor): 1st concentration parameter of the distribution
(often referred to as alpha)
concentration0 (float or Tensor): 2nd concentration parameter of the distribution
(often referred to as beta)
"""
arg_constraints = {
'concentration1': constraints.positive,
'concentration0': constraints.positive
}
support = constraints.unit_interval
has_rsample = True
def __init__(self, concentration1, concentration0, validate_args=None):
if isinstance(concentration1, Number) and isinstance(
concentration0, Number):
concentration1_concentration0 = torch.tensor(
[float(concentration1),
float(concentration0)])
else:
concentration1, concentration0 = broadcast_all(
concentration1, concentration0)
concentration1_concentration0 = torch.stack(
[concentration1, concentration0], -1)
self._dirichlet = Dirichlet(concentration1_concentration0)
super(Beta, self).__init__(self._dirichlet._batch_shape,
validate_args=validate_args)
@property
def mean(self):
return self.concentration1 / (self.concentration1 +
self.concentration0)
@property
def variance(self):
total = self.concentration1 + self.concentration0
return (self.concentration1 * self.concentration0 / (total.pow(2) *
(total + 1)))
def rsample(self, sample_shape=()):
value = self._dirichlet.rsample(sample_shape).select(-1, 0)
if isinstance(value, Number):
value = self._dirichlet.concentration.new_tensor(value)
return value
def log_prob(self, value):
if self._validate_args:
self._validate_sample(value)
heads_tails = torch.stack([value, 1.0 - value], -1)
return self._dirichlet.log_prob(heads_tails)
def entropy(self):
return self._dirichlet.entropy()
@property
def concentration1(self):
result = self._dirichlet.concentration[..., 0]
if isinstance(result, Number):
return torch.Tensor([result])
else:
return result
@property
def concentration0(self):
result = self._dirichlet.concentration[..., 1]
if isinstance(result, Number):
return torch.Tensor([result])
else:
return result
@property
def _natural_params(self):
return (self.concentration1, self.concentration0)
def _log_normalizer(self, x, y):
return torch.lgamma(x) + torch.lgamma(y) - torch.lgamma(x + y)
class Beta(ExponentialFamily):
r"""
Beta distribution parameterized by `concentration1` and `concentration0`.
Example::
>>> m = Beta(torch.tensor([0.5]), torch.tensor([0.5]))
>>> m.sample() # Beta distributed with concentration concentration1 and concentration0
tensor([ 0.1046])
Args:
concentration1 (float or Tensor): 1st concentration parameter of the distribution
(often referred to as alpha)
concentration0 (float or Tensor): 2nd concentration parameter of the distribution
(often referred to as beta)
"""
arg_constraints = {'concentration1': constraints.positive, 'concentration0': constraints.positive}
support = constraints.unit_interval
has_rsample = True
def __init__(self, concentration1, concentration0, validate_args=None):
if isinstance(concentration1, Number) and isinstance(concentration0, Number):
concentration1_concentration0 = torch.tensor([float(concentration1), float(concentration0)])
else:
concentration1, concentration0 = broadcast_all(concentration1, concentration0)
concentration1_concentration0 = torch.stack([concentration1, concentration0], -1)
self._dirichlet = Dirichlet(concentration1_concentration0)
super(Beta, self).__init__(self._dirichlet._batch_shape, validate_args=validate_args)
@property
def mean(self):
return self.concentration1 / (self.concentration1 + self.concentration0)
@property
def variance(self):
total = self.concentration1 + self.concentration0
return (self.concentration1 * self.concentration0 /
(total.pow(2) * (total + 1)))
def rsample(self, sample_shape=()):
value = self._dirichlet.rsample(sample_shape).select(-1, 0)
if isinstance(value, Number):
value = self._dirichlet.concentration.new_tensor(value)
return value
def log_prob(self, value):
if self._validate_args:
self._validate_sample(value)
heads_tails = torch.stack([value, 1.0 - value], -1)
return self._dirichlet.log_prob(heads_tails)
def entropy(self):
return self._dirichlet.entropy()
@property
def concentration1(self):
result = self._dirichlet.concentration[..., 0]
if isinstance(result, Number):
return torch.tensor([result])
else:
return result
@property
def concentration0(self):
result = self._dirichlet.concentration[..., 1]
if isinstance(result, Number):
return torch.tensor([result])
else:
return result
@property
def _natural_params(self):
return (self.concentration1, self.concentration0)
def _log_normalizer(self, x, y):
return torch.lgamma(x) + torch.lgamma(y) - torch.lgamma(x + y)
def forward(self, alpha, value):
dirichlet = TorchDirichlet(alpha)
return dirichlet.log_prob(value)
class ZOIBeta(ExponentialFamily):
""" Zero one inflated Beta distribution
Args:
p (float or Tensor): Pr(y = 0)
q (float or Tensor): Pr(y = 1 | y != 0)
concentration1 (float or Tensor): 1st Beta dist. parameter
(often referred to as alpha)
concentration0 (float or Tensor): 2nd Beta dist. parameter
(often referred to as beta)
"""
arg_constraints = {
'p': constraints.unit_interval,
'q': constraints.unit_interval,
'concentration1': constraints.positive,
'concentration0': constraints.positive
}
support = constraints.unit_interval # does this include 0 and 1?
has_rsample = False
def __init__(self,
p,
q,
concentration1,
concentration0,
validate_args=None):
if isinstance(concentration1, Number) and isinstance(
concentration0, Number):
self.concentration1_concentration0 = torch.tensor(
[float(concentration1),
float(concentration0)])
else:
concentration1, concentration0 = broadcast_all(
concentration1, concentration0)
self.concentration1_concentration0 = torch.stack(
[concentration1, concentration0], -1)
self._dirichlet = Dirichlet(self.concentration1_concentration0)
self.log_p = torch.log(p)
self.log1m_p = torch.log(1 - p)
self.log_q = torch.log(q)
self.log1m_q = torch.log(1 - q)
super(ZOIBeta, self).__init__(self._dirichlet._batch_shape,
validate_args=validate_args)
def beta_lp(self, value):
if self._validate_args:
self._validate_sample(value)
heads_tails = torch.stack([value, 1.0 - value], -1)
return self._dirichlet.log_prob(heads_tails)
def log_prob(self, value):
lp = torch.zeros_like(value, dtype=torch.float)
if torch.mul(value > 0., value < 1.).any():
beta_idx = torch.where(torch.mul(value > 0., value < 1.))
self._dirichlet = Dirichlet(
self.concentration1_concentration0[beta_idx])
lp[beta_idx] = self.log1m_p[beta_idx] + self.log1m_q[
beta_idx] + self.beta_lp(value[beta_idx])
lp[torch.where(value == 0.)] = self.log_p[torch.where(value == 0.)]
lp[torch.where(value == 1.)] = self.log1m_p[torch.where(
value == 1.)] + self.log_q[torch.where(value == 1.)]
return lp
class Beta(Distribution):
r"""
Beta distribution parameterized by `concentration1` and `concentration0`.
Example::
>>> m = Beta(torch.Tensor([0.5]), torch.Tensor([0.5]))
>>> m.sample() # Beta distributed with concentration concentration1 and concentration0
0.1046
[torch.FloatTensor of size 1]
Args:
concentration1 (float or Tensor or Variable): 1st concentration parameter of the distribution
(often referred to as alpha)
concentration0 (float or Tensor or Variable): 2nd concentration parameter of the distribution
(often referred to as beta)
"""
params = {'concentration1': constraints.positive, 'concentration0': constraints.positive}
support = constraints.unit_interval
has_rsample = True
def __init__(self, concentration1, concentration0):
if isinstance(concentration1, Number) and isinstance(concentration0, Number):
concentration1_concentration0 = variable([concentration1, concentration0])
else:
concentration1, concentration0 = broadcast_all(concentration1, concentration0)
concentration1_concentration0 = torch.stack([concentration1, concentration0], -1)
self._dirichlet = Dirichlet(concentration1_concentration0)
super(Beta, self).__init__(self._dirichlet._batch_shape)
@property
def mean(self):
return self.concentration1 / (self.concentration1 + self.concentration0)
@property
def variance(self):
total = self.concentration1 + self.concentration0
return (self.concentration1 * self.concentration0 /
(total.pow(2) * (total + 1)))
def rsample(self, sample_shape=()):
value = self._dirichlet.rsample(sample_shape).select(-1, 0)
if isinstance(value, Number):
value = self._dirichlet.concentration.new([value])
return value
def log_prob(self, value):
self._validate_log_prob_arg(value)
heads_tails = torch.stack([value, 1.0 - value], -1)
return self._dirichlet.log_prob(heads_tails)
def entropy(self):
return self._dirichlet.entropy()
@property
def concentration1(self):
result = self._dirichlet.concentration[..., 0]
if isinstance(result, Number):
return torch.Tensor([result])
else:
return result
@property
def concentration0(self):
result = self._dirichlet.concentration[..., 1]
if isinstance(result, Number):
return torch.Tensor([result])
else:
return result
class Beta(ExponentialFamily):
r"""
Beta distribution parameterized by :attr:`concentration1` and :attr:`concentration0`.
Example::
>>> # xdoctest: +IGNORE_WANT("non-deterinistic")
>>> m = Beta(torch.tensor([0.5]), torch.tensor([0.5]))
>>> m.sample() # Beta distributed with concentration concentration1 and concentration0
tensor([ 0.1046])
Args:
concentration1 (float or Tensor): 1st concentration parameter of the distribution
(often referred to as alpha)
concentration0 (float or Tensor): 2nd concentration parameter of the distribution
(often referred to as beta)
"""
arg_constraints = {
'concentration1': constraints.positive,
'concentration0': constraints.positive
}
support = constraints.unit_interval
has_rsample = True
def __init__(self, concentration1, concentration0, validate_args=None):
if isinstance(concentration1, Real) and isinstance(
concentration0, Real):
concentration1_concentration0 = torch.tensor(
[float(concentration1),
float(concentration0)])
else:
concentration1, concentration0 = broadcast_all(
concentration1, concentration0)
concentration1_concentration0 = torch.stack(
[concentration1, concentration0], -1)
self._dirichlet = Dirichlet(concentration1_concentration0,
validate_args=validate_args)
super(Beta, self).__init__(self._dirichlet._batch_shape,
validate_args=validate_args)
def expand(self, batch_shape, _instance=None):
new = self._get_checked_instance(Beta, _instance)
batch_shape = torch.Size(batch_shape)
new._dirichlet = self._dirichlet.expand(batch_shape)
super(Beta, new).__init__(batch_shape, validate_args=False)
new._validate_args = self._validate_args
return new
@property
def mean(self):
return self.concentration1 / (self.concentration1 +
self.concentration0)
@property
def mode(self):
return self._dirichlet.mode[..., 0]
@property
def variance(self):
total = self.concentration1 + self.concentration0
return (self.concentration1 * self.concentration0 / (total.pow(2) *
(total + 1)))
def rsample(self, sample_shape=()):
return self._dirichlet.rsample(sample_shape).select(-1, 0)
def log_prob(self, value):
if self._validate_args:
self._validate_sample(value)
heads_tails = torch.stack([value, 1.0 - value], -1)
return self._dirichlet.log_prob(heads_tails)
def entropy(self):
return self._dirichlet.entropy()
@property
def concentration1(self):
result = self._dirichlet.concentration[..., 0]
if isinstance(result, Number):
return torch.tensor([result])
else:
return result
@property
def concentration0(self):
result = self._dirichlet.concentration[..., 1]
if isinstance(result, Number):
return torch.tensor([result])
else:
return result
@property
def _natural_params(self):
return (self.concentration1, self.concentration0)
def _log_normalizer(self, x, y):
return torch.lgamma(x) + torch.lgamma(y) - torch.lgamma(x + y)