def __getitem__(self, idx):
# idx only acts as a counter while generating batches.
seq_len = torch.randint(self.min_seq_len,
self.max_seq_len, (1, ),
dtype=torch.long).item()
prob = 0.5 * torch.ones([seq_len, self.seq_width], dtype=torch.float64)
seq = Binomial(1, prob).sample()
# seq_len = torch.randint(
# self.min_seq_len, self.max_seq_len, (1,), dtype=torch.long).item().to(self.device)
# prob = 0.5 * torch.ones([seq_len, self.seq_width], dtype=torch.float64).to(self.device)
# seq = Binomial(1, prob).sample().to(self.device)
# inverse the input
idx = [i for i in range(seq.size(0) - 1, -1, -1)]
idx = torch.LongTensor(idx)
inverted_tensor = seq.index_select(0, idx)
# fill in input sequence, two bit longer and wider than target
# input_seq = torch.zeros([seq_len + 2, self.seq_width + 2]).to(self.device)
input_seq = torch.zeros([seq_len + 2, self.seq_width + 2])
input_seq[0, self.seq_width] = 1.0 # start delimiter
input_seq[1:seq_len + 1, :self.seq_width] = seq
input_seq[seq_len + 1, self.seq_width + 1] = 1.0 # end delimiter
# target_seq = torch.zeros([seq_len, self.seq_width]).to(self.device)
target_seq = torch.zeros([seq_len, self.seq_width])
target_seq[:seq_len, :self.seq_width] = inverted_tensor
return {'input': input_seq, 'target': target_seq}
def create_subsequence_mask(o, r=.15, lm=3, stateful=True, sync=False):
device = o.device
if o.ndim == 2: o = o[None]
n_masks, mask_dims, mask_len = o.shape
if sync == 'random': sync = random.random() > .5
dims = 1 if sync else mask_dims
if stateful:
numels = n_masks * dims * mask_len
pm = torch.tensor([1 / lm], device=device)
pu = torch.clip(pm * (r / max(1e-6, 1 - r)), 1e-3, 1)
zot, proba_a, proba_b = (torch.as_tensor([False, True], device=device), pu, pm) if random.random() > pm else \
(torch.as_tensor([True, False], device=device), pm, pu)
max_len = max(1, 2 * math.ceil(numels // (1/pm + 1/pu)))
for i in range(10):
_dist_a = (Geometric(probs=proba_a).sample([max_len])+1).long()
_dist_b = (Geometric(probs=proba_b).sample([max_len])+1).long()
dist_a = _dist_a if i == 0 else torch.cat((dist_a, _dist_a), dim=0)
dist_b = _dist_b if i == 0 else torch.cat((dist_b, _dist_b), dim=0)
add = torch.add(dist_a, dist_b)
if torch.gt(torch.sum(add), numels): break
dist_len = torch.argmax((torch.cumsum(add, 0) >= numels).float()) + 1
if dist_len%2: dist_len += 1
repeats = torch.cat((dist_a[:dist_len], dist_b[:dist_len]), -1).flatten()
zot = zot.repeat(dist_len)
mask = torch.repeat_interleave(zot, repeats)[:numels].reshape(n_masks, dims, mask_len)
else:
probs = torch.tensor(r, device=device)
mask = Binomial(1, probs).sample((n_masks, dims, mask_len)).bool()
if sync: mask = mask.repeat(1, mask_dims, 1)
return mask
def main(args):
np.random.seed(args.seed)
torch.manual_seed(args.seed)
n, d = args.n, args.d
p = np.sqrt(d) / d
probs = torch.ones(d) * p
sampler = Binomial(1, probs)
data = sampler.sample(sample_shape=(torch.Size([n])))
# add some correlation
offset = int(np.sqrt(d))
for i in range(offset):
data[:, i + offset] = data[:, i]
weight = torch.randn((d))
noise = torch.randn((n)) / 2
labels = data @ weight + noise
print('data shape', data.shape)
path = os.path.join(args.save_prefix, 'sparse_' + str(n) + '_' + str(d))
os.makedirs(path)
data_path = os.path.join(path, 'data.npy')
p_path = os.path.join(path, 'p.npy')
labels_path = os.path.join(path, 'labels.npy')
np.save(data_path, data.numpy())
np.save(p_path, p)
np.save(labels_path, labels.numpy())
def test3():
"""
The binomial distribution
:return:
"""
from torch.distributions.binomial import Binomial
# 100-count of trials: 0, 0.2, 0.8 and 1 are event probabilities
dist = Binomial(100, torch.tensor([0, 0.2, 0.8, 1]))
dist.sample() # tensor([ 0., 19., 72., 100.])
def create_future_mask(o, r=.15, sync=False):
if o.ndim == 2: o = o[None]
n_masks, mask_dims, mask_len = o.shape
if sync == 'random': sync = random.random() > .5
dims = 1 if sync else mask_dims
probs = torch.tensor(r, device=o.device)
mask = Binomial(1, probs).sample((n_masks, dims, mask_len))
if sync: mask = mask.repeat(1, mask_dims, 1)
mask = torch.sort(mask,dim=-1, descending=True)[0].bool()
return mask
def __init__(self,
total_count=1,
probs=None,
logits=None,
validate_args=None):
if not isinstance(total_count, int):
raise NotImplementedError(
'inhomogeneous total_count is not supported')
self.total_count = total_count
self._categorical = Categorical(probs=probs, logits=logits)
self._binomial = Binomial(total_count=total_count, probs=self.probs)
batch_shape = self._categorical.batch_shape
event_shape = self._categorical.param_shape[-1:]
super(Multinomial, self).__init__(batch_shape,
event_shape,
validate_args=validate_args)
def get_sample_wlen(self, bs=1):
# idx only acts as a counter while generating batches.
prob = 0.5 * torch.ones([self.input_seq_len, bs, self.seq_width],
dtype=torch.float64)
seq = Binomial(1, prob).sample()
# Extra input channel for providing priority value
input_seq = torch.zeros([self.input_seq_len, bs, self.in_dim])
input_seq[:self.input_seq_len, :, :self.seq_width] = seq
# torch's Uniform function draws samples from the half-open interval
# [low, high) but in the paper the priorities are drawn from [-1,1].
# This minor difference is being ignored here as supposedly it doesn't
# affects the task.
priority = Uniform(torch.tensor([-1.0] * bs), torch.tensor([1.0] * bs))
for i in range(self.input_seq_len):
input_seq[i, :, self.seq_width] = priority.sample()
target_seq = []
for j in range(bs):
sorted, ind = torch.sort(input_seq[:, j, self.seq_width],
0,
descending=True)
sorted = input_seq[ind, j]
target_seq.append(
sorted[:self.target_seq_len, :self.seq_width].unsqueeze(1))
target_seq = torch.cat(target_seq, 1)
return {'input': input_seq, 'target': target_seq}
def __getitem__(self, idx):
# idx only acts as a counter while generating batches.
prob = 0.5 * torch.ones([self.input_seq_len, self.seq_width],
dtype=torch.float64)
seq = Binomial(1, prob).sample()
# Extra input channel for providing priority value
input_seq = torch.zeros([self.input_seq_len, self.seq_width + 1])
input_seq[:self.input_seq_len, :self.seq_width] = seq
# torch's Uniform function draws samples from the half-open interval
# [low, high) but in the paper the priorities are drawn from [-1,1].
# This minor difference is being ignored here as supposedly it doesn't
# affects the task.
if not self.uniform:
alpha = torch.tensor([2.0])
beta = torch.tensor([5.0])
if self.random_distr:
alpha_beta_gen = Uniform(torch.tensor([0.0]),
torch.tensor([100.0]))
alpha = alpha_beta_gen.sample()
beta = alpha_beta_gen.sample()
priority = Beta(alpha, beta)
else:
priority = Uniform(torch.tensor([-1.0]), torch.tensor([1.0]))
for i in range(self.input_seq_len):
input_seq[i, self.seq_width] = priority.sample()
sorted_index = torch.sort(input_seq[:, -1], descending=True)[1]
target_seq = input_seq[sorted_index][:self.target_seq_len, :self.
seq_width]
return {'input': input_seq, 'target': target_seq}
def __getitem__(self, idx):
# idx only acts as a counter while generating batches.
num_item = torch.randint(self.min_item,
self.max_item, (1, ),
dtype=torch.long).item()
prob = 0.5 * \
torch.ones([self.seq_len, self.seq_width], dtype=torch.float64)
seq = Binomial(1, prob)
# fill in input two bit wider than target to account for delimiter
# flags.
input_items = torch.zeros([(self.seq_len + 1) * (num_item + 1) + 1,
self.seq_width + 2])
for i in range(num_item):
input_items[(self.seq_len + 1) * i, self.seq_width] = 1.0
input_items[(self.seq_len + 1) * i + 1:(self.seq_len + 1) *
(i + 1), :self.seq_width] = seq.sample()
# generate query item randomly
# in case of only one item, torch.randint throws error as num_item-1=0
query_item = 0
if num_item != 1:
query_item = torch.randint(0,
num_item - 1, (1, ),
dtype=torch.long).item()
query_seq = input_items[(self.seq_len + 1) * query_item +
1:(self.seq_len + 1) *
(query_item + 1), :self.seq_width]
input_items[(self.seq_len + 1) * num_item,
self.seq_width + 1] = 1.0 # query delimiter
input_items[(self.seq_len + 1) * num_item + 1:(self.seq_len + 1) *
(num_item + 1), :self.seq_width] = query_seq
input_items[(self.seq_len + 1) * (num_item + 1),
self.seq_width + 1] = 1.0 # query delimiter
# generate target sequences(item next to query in the input list)
target_item = torch.zeros([self.seq_len, self.seq_width])
# in case of last item, target sequence is zero
if query_item != num_item - 1:
target_item[:self.seq_len, :self.seq_width] = input_items[
(self.seq_len + 1) * (query_item + 1) + 1:(self.seq_len + 1) *
(query_item + 2), :self.seq_width]
return {'input': input_items, 'target': target_item}
def forward(self, x):
# add noise
x = x * Binomial(total_count=1,
probs=1 - self.corruption_level).sample([x.shape[1]])
x = self.encoder(x)
x = self.decoder(x)
return x
def get_sample_wlen(self, seq_len, bs=1):
rep = torch.randint(self.min_repeat,
self.max_repeat, (1, ),
dtype=torch.long).item()
prob = 0.5 * torch.ones([seq_len, bs, self.seq_width],
dtype=torch.float64)
seq = Binomial(1, prob).sample()
# fill in input sequence, two bit longer and wider than target
input_seq = torch.zeros([seq_len + 2, bs, self.seq_width + 2])
input_seq[0, :, self.seq_width] = 1.0 # delimiter
input_seq[1:seq_len + 1, :, :self.seq_width] = seq
input_seq[seq_len + 1, :, self.seq_width + 1] = self.normalise(rep)
target_seq = torch.zeros([seq_len * rep + 1, bs, self.seq_width + 1])
target_seq[:seq_len * rep, :, :self.seq_width] = seq.repeat(rep, 1, 1)
target_seq[seq_len * rep, :, self.seq_width] = 1.0 # delimiter
return {'input': input_seq, 'target': target_seq}
def forward(self, x):
success_values = torch.arange(0,
self.num_classes,
dtype=x.dtype,
device=x.device,
requires_grad=False).repeat(
x.size(0), 1)
succes_prob = self.prob_output(x).repeat(1, self.num_classes)
scores = Binomial(total_count=self.num_classes - 1,
probs=succes_prob).log_prob(success_values)
scores = torch.softmax(scores / self.tau, dim=-1)
return scores
def simulate(self, theta, psi):
# theta = [beta, gamma]
# psi = tau
# sample = [S(tau), I(tau), R(tau)]
beta = theta[0].item()
gamma = theta[1].item()
psi = psi.item()
S = self.population_size - 1
I = 1
R = 0
n_steps = int(psi / self.step_size)
for i in range(n_steps):
if I == 0: # State will remain the same.
break
delta_I = int(
Binomial(S, beta * I / self.population_size).sample())
delta_R = int(Binomial(I, gamma).sample())
S -= delta_I
I = I + delta_I - delta_R
R += delta_R
return torch.tensor([S, I, R]).float()
def __getitem__(self, idx):
# idx only acts as a counter while generating batches.
seq_len = torch.randint(self.min_seq_len,
self.max_seq_len, (1, ),
dtype=torch.long).item()
rep = torch.randint(self.min_repeat,
self.max_repeat, (1, ),
dtype=torch.long).item()
prob = 0.5 * torch.ones([seq_len, self.seq_width], dtype=torch.float64)
seq = Binomial(1, prob).sample()
# fill in input sequence, two bit longer and wider than target
input_seq = torch.zeros([seq_len + 2, self.seq_width + 2])
input_seq[0, self.seq_width] = 1.0 # delimiter
input_seq[1:seq_len + 1, :self.seq_width] = seq
input_seq[seq_len + 1, self.seq_width + 1] = self.normalise(rep)
target_seq = torch.zeros([seq_len * rep + 1, self.seq_width + 1])
target_seq[:seq_len * rep, :self.seq_width] = seq.repeat(rep, 1)
target_seq[seq_len * rep, self.seq_width] = 1.0 # delimiter
return {'input': input_seq, 'target': target_seq}
def data_params():
"""
Returns random data and parameters used to generate the data.
"""
S = int(1e5)
theta_1 = 0.5
theta_2 = 0.3
pi_1 = 0.6
pi_2 = 1 - pi_1
N_ls_all = t.FloatTensor([100 for _ in range(S)])
theta_ls = t.FloatTensor(
np.random.choice([theta_1, theta_2], size=S, p=[pi_1, pi_2]))
n_ls_all = Binomial(N_ls_all, theta_ls).sample()
return N_ls_all, n_ls_all, [pi_1, pi_2, theta_1, theta_2]
def generate_mutation_maps(self, action):
"""
Create the binary map describing how the chromosome of each individual is mutated.
For each individual, the probability of one chromosome flipping is defined
by the sampled action.
"""
genome_length = self.encoding_strategy.problem.num_dimensions
# Expand action, so that all chromosomes of an individual
# have the same mutation probability
binomial_probs = action.view(-1).unsqueeze(1).expand(-1, genome_length)
return Binomial(1, probs=binomial_probs).sample().cpu().numpy()
def get_sample_wlen(self, seq_len, bs=1):
# idx only acts as a counter while generating batches.
prob = 0.5 * torch.ones([seq_len, bs, self.seq_width],
dtype=torch.float64)
seq = Binomial(1, prob).sample()
# fill in input sequence, two bit longer and wider than target
input_seq = torch.zeros([seq_len + 2, bs, self.seq_width + 2])
input_seq[0, :, self.seq_width] = 1.0 # start delimiter
input_seq[1:seq_len + 1, :, :self.seq_width] = seq
input_seq[seq_len + 1, :, self.seq_width + 1] = 1.0 # end delimiter
target_seq = torch.zeros([seq_len, bs, self.seq_width])
target_seq[:seq_len, :, :self.seq_width] = seq
return {'input': input_seq, 'target': target_seq}
def __init__(self, model_conf, num_users, num_items, device):
super(DAE, self).__init__()
self.hidden_dim = model_conf.hidden_dim
self.act = model_conf.act
self.corruption_ratio = model_conf.corruption_ratio
self.num_users = num_users
self.num_items = num_items
self.binomial = Binomial(total_count=1,
probs=(1 - self.corruption_ratio))
self.device = device
self.encoder = nn.Linear(self.num_items, self.hidden_dim)
self.decoder = nn.Linear(self.hidden_dim, self.num_items)
self.to(self.device)
def simulate(self, theta, psi):
# theta = [beta, gamma]
# psi = tau
# sample = [S(tau), I(tau), R(tau)]
infection_rate = theta.item()
design = psi.item()
I = 0
t = 0.0
n_steps = int(psi / self.step_size)
for _ in range(n_steps):
S = self.population_size - I
if S == 0:
break
p_inf = 1 - np.exp(-infection_rate * t)
delta_I = int(Binomial(S, p_inf).sample())
I += delta_I
t += self.step_size
return torch.tensor(I).float()
def __getitem__(self, idx):
# Get sequence length
seq_len = torch.randint(self.min_seq_len,
self.max_seq_len, (1, ),
dtype=torch.long).item()
# Generate sequences
prob = 0.5 * torch.ones([seq_len, self.seq_width], dtype=torch.float64)
seq = Binomial(1, prob).sample()
# Fill in input sequence, two bit longer and wider than target
input_seq = torch.zeros([seq_len + 2, self.seq_width + 2])
input_seq[0, self.seq_width] = 1.0 # start delimiter
input_seq[1:seq_len + 1, :self.seq_width] = seq
input_seq[seq_len + 1, self.seq_width + 1] = 1.0 # end delimiter
# Create target sequence
target_seq = torch.zeros([seq_len, self.seq_width])
target_seq[:seq_len, :self.seq_width] = seq
return {'input': input_seq, 'target': target_seq}
def __init__(self, model_conf, num_users, num_items, device):
"""
:param model_conf: model configuration
:param num_users: number of users
:param num_items: number of items
:param device: choice of device
"""
super(DAE, self).__init__()
self.hidden_dim = model_conf.hidden_dim
self.act = model_conf.act
self.corruption_ratio = model_conf.corruption_ratio
self.num_users = num_users
self.num_items = num_items
self.binomial = Binomial(total_count=1, probs=(1 - self.corruption_ratio))
self.device = device
self.encoder = nn.Linear(self.num_items, self.hidden_dim)
self.decoder = nn.Linear(self.hidden_dim, self.num_items)
self.to(self.device)
def __getitem__(self, idx):
# idx only acts as a counter while generating batches.
prob = 0.5 * torch.ones([self.input_seq_len, self.seq_width],
dtype=torch.float64)
seq = Binomial(1, prob).sample()
# Extra input channel for providing priority value
input_seq = torch.zeros([self.input_seq_len, self.seq_width + 1])
input_seq[:self.input_seq_len, :self.seq_width] = seq
# torch's Uniform function draws samples from the half-open interval
# [low, high) but in the paper the priorities are drawn from [-1,1].
# This minor difference is being ignored here as supposedly it doesn't
# affects the task.
priority = Uniform(torch.tensor([-1.0]), torch.tensor([1.0]))
for i in range(self.input_seq_len):
input_seq[i, self.seq_width] = priority.sample()
sorted, _ = torch.sort(input_seq, 0, descending=True)
target_seq = sorted[:self.target_seq_len, :self.seq_width]
return {'input': input_seq, 'target': target_seq}
def calc_logL(self, N_ls, n_ls):
'''
Calculate the log likelihood.
Input
-----
N_ls: a [S] shape tensor = [N1, N2, ..., NS]
n_ls: a [S] shape tensor = [n1, n2, ..., nS]
Output
------
log_likelihood: the conditional probability of the parameters
(pi and theta) given the observed data (N, n)
'''
S = len(N_ls)
pi_list = self.pi_list
theta_list = self.theta_list
#K = self.K
# log_binom_mat has shape (S,K), element_{i,l} = log_Binomial(ni|Ni, theta_l)
# log with natural base.
log_binom_mat = Binomial(N_ls.reshape(S, 1),
theta_list.reshape(1, self.K)).log_prob(
n_ls.reshape(S, 1))
# mean_log_binom, the mean value of all elements in log_binom_mat.
c = torch.mean(log_binom_mat)
# binom_mat has shape (S,K), element_{i,l} = Binomial(ni|Ni, theta_l)
binom_mat = torch.exp(log_binom_mat - c)
# log_likelihood = sum_{i=1}^{S} log(prob_i), this is a real number
log_likelihood = S * c + torch.sum(
torch.log(torch.matmul(binom_mat, pi_list)))
return log_likelihood
def get_sample_wlen(self, num_item, seq_len, bs=1):
prob = 0.5 * \
torch.ones([seq_len, bs, self.seq_width], dtype=torch.float64)
seq = Binomial(1, prob)
# fill in input two bit wider than target to account for delimiter
# flags.
input_items = torch.zeros([(seq_len + 1) * (num_item + 1) + 1, bs,
self.seq_width + 2])
for i in range(num_item):
input_items[(seq_len + 1) * i, :, self.seq_width] = 1.0
input_items[(seq_len + 1) * i + 1:(seq_len + 1) *
(i + 1), :, :self.seq_width] = seq.sample()
# generate query item randomly
# in case of only one item, torch.randint throws error as num_item-1=0
query_item = 0
if num_item != 1:
query_item = torch.randint(0,
num_item - 1, (1, ),
dtype=torch.long).item()
query_seq = input_items[(seq_len + 1) * query_item + 1:(seq_len + 1) *
(query_item + 1), :, :self.seq_width]
input_items[(seq_len + 1) * num_item, :,
self.seq_width + 1] = 1.0 # query delimiter
input_items[(seq_len + 1) * num_item + 1:(seq_len + 1) *
(num_item + 1), :, :self.seq_width] = query_seq
input_items[(seq_len + 1) * (num_item + 1), :,
self.seq_width + 1] = 1.0 # query delimiter
# generate target sequences(item next to query in the input list)
target_item = torch.zeros([seq_len, bs, self.seq_width])
# in case of last item, target sequence is zero
for b in range(bs):
choose_max = False
qitem = input_items[(seq_len + 1) * query_item + 1:(seq_len + 1) *
(query_item + 1), b, :self.seq_width]
if qitem.contiguous().view(-1)[-1].item() == 1:
choose_max = True
# print("max")
else:
pass
# print("min")
if choose_max:
cd = 0
else:
cd = 10000000000000000
titem = qitem
for ii in range(num_item):
if ii != query_item:
cur_item = input_items[(seq_len + 1) * ii +
1:(seq_len + 1) * (ii + 1),
b, :self.seq_width]
curd = torch.norm(qitem.contiguous().view(-1) -
cur_item.contiguous().view(-1))
if choose_max:
if curd > cd:
titem = ii
cd = curd
else:
if curd < cd:
titem = ii
cd = curd
# print(num_item)
# print(titem)
# print(cd)
target_item[:seq_len, b, :self.seq_width] = input_items[
(seq_len + 1) * titem + 1:(seq_len + 1) * (titem + 1),
b, :self.seq_width]
return {'input': input_items, 'target': target_item}
def _sampleWidthByAlphas(self):
# define Binomial distribution on n-1 layer filters (because we have to choose at least one filter)
dist = Binomial(self.outputChannels() - 1, logits=self._alphas)
# draw from distribution
width = 1 + dist.sample().type(int32).item()
return width
def calcNewWidthFunc(width: int,
alphaWidth: AlphaPerWidthBlock.AlphaWidth):
# define Binomial distribution on n-1 layer filters (because we have to choose at least one filter)
dist = Binomial(width - 1, logits=alphaWidth.tensor())
# draw from distribution
return 1 + dist.sample().type(int32).item()
def create_distribution(self, distribution_params):
return Binomial(1, logits=distribution_params)
class Multinomial(Distribution):
r"""
Creates a Multinomial distribution parameterized by :attr:`total_count` and
either :attr:`probs` or :attr:`logits` (but not both). The innermost dimension of
:attr:`probs` indexes over categories. All other dimensions index over batches.
Note that :attr:`total_count` need not be specified if only :meth:`log_prob` is
called (see example below)
.. note:: The `probs` argument must be non-negative, finite and have a non-zero sum,
and it will be normalized to sum to 1 along the last dimension. :attr:`probs`
will return this normalized value.
The `logits` argument will be interpreted as unnormalized log probabilities
and can therefore be any real number. It will likewise be normalized so that
the resulting probabilities sum to 1 along the last dimension. :attr:`logits`
will return this normalized value.
- :meth:`sample` requires a single shared `total_count` for all
parameters and samples.
- :meth:`log_prob` allows different `total_count` for each parameter and
sample.
Example::
>>> m = Multinomial(100, torch.tensor([ 1., 1., 1., 1.]))
>>> x = m.sample() # equal probability of 0, 1, 2, 3
tensor([ 21., 24., 30., 25.])
>>> Multinomial(probs=torch.tensor([1., 1., 1., 1.])).log_prob(x)
tensor([-4.1338])
Args:
total_count (int): number of trials
probs (Tensor): event probabilities
logits (Tensor): event log probabilities (unnormalized)
"""
arg_constraints = {
'probs': constraints.simplex,
'logits': constraints.real_vector
}
total_count: int
@property
def mean(self):
return self.probs * self.total_count
@property
def variance(self):
return self.total_count * self.probs * (1 - self.probs)
def __init__(self,
total_count=1,
probs=None,
logits=None,
validate_args=None):
if not isinstance(total_count, int):
raise NotImplementedError(
'inhomogeneous total_count is not supported')
self.total_count = total_count
self._categorical = Categorical(probs=probs, logits=logits)
self._binomial = Binomial(total_count=total_count, probs=self.probs)
batch_shape = self._categorical.batch_shape
event_shape = self._categorical.param_shape[-1:]
super(Multinomial, self).__init__(batch_shape,
event_shape,
validate_args=validate_args)
def expand(self, batch_shape, _instance=None):
new = self._get_checked_instance(Multinomial, _instance)
batch_shape = torch.Size(batch_shape)
new.total_count = self.total_count
new._categorical = self._categorical.expand(batch_shape)
super(Multinomial, new).__init__(batch_shape,
self.event_shape,
validate_args=False)
new._validate_args = self._validate_args
return new
def _new(self, *args, **kwargs):
return self._categorical._new(*args, **kwargs)
@constraints.dependent_property(is_discrete=True, event_dim=1)
def support(self):
return constraints.multinomial(self.total_count)
@property
def logits(self):
return self._categorical.logits
@property
def probs(self):
return self._categorical.probs
@property
def param_shape(self):
return self._categorical.param_shape
def sample(self, sample_shape=torch.Size()):
sample_shape = torch.Size(sample_shape)
samples = self._categorical.sample(
torch.Size((self.total_count, )) + sample_shape)
# samples.shape is (total_count, sample_shape, batch_shape), need to change it to
# (sample_shape, batch_shape, total_count)
shifted_idx = list(range(samples.dim()))
shifted_idx.append(shifted_idx.pop(0))
samples = samples.permute(*shifted_idx)
counts = samples.new(self._extended_shape(sample_shape)).zero_()
counts.scatter_add_(-1, samples, torch.ones_like(samples))
return counts.type_as(self.probs)
def entropy(self):
n = torch.tensor(self.total_count)
cat_entropy = self._categorical.entropy()
term1 = n * cat_entropy - torch.lgamma(n + 1)
support = self._binomial.enumerate_support(expand=False)[1:]
binomial_probs = torch.exp(self._binomial.log_prob(support))
weights = torch.lgamma(support + 1)
term2 = (binomial_probs * weights).sum([0, -1])
return term1 + term2
def log_prob(self, value):
if self._validate_args:
self._validate_sample(value)
logits, value = broadcast_all(self.logits, value)
logits = logits.clone(memory_format=torch.contiguous_format)
log_factorial_n = torch.lgamma(value.sum(-1) + 1)
log_factorial_xs = torch.lgamma(value + 1).sum(-1)
logits[(value == 0) & (logits == -inf)] = 0
log_powers = (logits * value).sum(-1)
return log_factorial_n - log_factorial_xs + log_powers
# In[141]: dist = Beta(torch.tensor([0.5]), torch.tensor([0.5])) dist # In[142]: dist.sample() # In[143]: from torch.distributions.binomial import Binomial # In[144]: dist = Binomial(100, torch.tensor([0, .2, .8, 1])) # In[147]: dist.sample() # In[148]: # 100- count of trials # 0, 0.2, 0.8 and 1 are event probabilities # In[174]: from torch.distributions.categorical import Categorical # In[175]:
model_path = os.path.join('checkpoints', model_name)
if not os.path.exists(model_path):
os.makedirs(model_path)
from tensorboardX import SummaryWriter
writer = SummaryWriter(os.path.join('runs', model_name))
n_iter = 0
num_samples = 50
sample = torch.randn(64, opts.nz).double().to(device)
for epoch in range(opts.epochs):
print('=> Epoch {}'.format(epoch))
model.train()
running_loss = []
for data in tqdm(train_loader):
image = data[0]
m = Binomial(1, image.view(-1, 784))
# inputs = m.sample(torch.Size([num_samples])).double().to(device)
inputs = m.sample().expand(num_samples, image.shape[0],
784).double().to(device)
optimizer.zero_grad()
loss, bce, kld = model.train_loss(inputs)
loss.backward()
optimizer.step()
running_loss.append(loss.item())
writer.add_scalar('bce', bce, n_iter)
writer.add_scalar('kld', kld, n_iter)
writer.add_scalar('loss', loss, n_iter)
n_iter += 1
writer.add_scalar('loss_epoch', np.mean(running_loss), epoch)