def __init__(self): super(BVAE, self).__init__() if torch.cuda.is_available(): self.dtype = torch.cuda.FloatTensor else: self.dtype = torch.FloatTensor self.z_size = 20 self.input_size = 784 #Encoder self.fc1 = nn.Linear(self.input_size, 200) self.fc2 = nn.Linear(200, self.z_size * 2) #Decoder # self.fc3 = nn.Linear(self.z_size, 200) # self.fc4 = nn.Linear(200, 784) # self.decoder = BNN([self.z_size, 200, 784], [torch.nn.Softplus, torch.nn.Softplus]) self.decoder = BNN([self.z_size, 200, 784], [F.relu, F.relu])
def __init__(self, qW_weight, seed=-1): super(BVAE, self).__init__() if seed != -1: torch.manual_seed(seed) if torch.cuda.is_available(): self.dtype = torch.cuda.FloatTensor else: self.dtype = torch.FloatTensor self.qW_weight = qW_weight self.z_size = 2 self.input_size = 784 #Encoder self.fc1 = nn.Linear(self.input_size, 200) self.fc2 = nn.Linear(200, self.z_size * 2) #Decoder self.decoder = BNN([self.z_size, 200, 784], [F.relu, F.relu])
def __init__(self): super(BVAE, self).__init__() if torch.cuda.is_available(): self.dtype = torch.cuda.FloatTensor else: self.dtype = torch.FloatTensor self.z_size = 20 self.input_size = 784 #Encoder self.fc1 = nn.Linear(self.input_size, 200) self.fc2 = nn.Linear(200, self.z_size*2) #Decoder # self.fc3 = nn.Linear(self.z_size, 200) # self.fc4 = nn.Linear(200, 784) # self.decoder = BNN([self.z_size, 200, 784], [torch.nn.Softplus, torch.nn.Softplus]) self.decoder = BNN([self.z_size, 200, 784], [F.relu, F.relu])
def __init__(self, qW_weight, seed=-1): super(BVAE, self).__init__() if seed != -1: torch.manual_seed(seed) if torch.cuda.is_available(): self.dtype = torch.cuda.FloatTensor else: self.dtype = torch.FloatTensor self.qW_weight = qW_weight self.z_size = 2 self.input_size = 784 #Encoder self.fc1 = nn.Linear(self.input_size, 200) self.fc2 = nn.Linear(200, self.z_size*2) #Decoder self.decoder = BNN([self.z_size, 200, 784], [F.relu, F.relu])
class BVAE(nn.Module): def __init__(self, qW_weight, seed=-1): super(BVAE, self).__init__() if seed != -1: torch.manual_seed(seed) if torch.cuda.is_available(): self.dtype = torch.cuda.FloatTensor else: self.dtype = torch.FloatTensor self.qW_weight = qW_weight self.z_size = 2 self.input_size = 784 #Encoder self.fc1 = nn.Linear(self.input_size, 200) self.fc2 = nn.Linear(200, self.z_size * 2) #Decoder self.decoder = BNN([self.z_size, 200, 784], [F.relu, F.relu]) def encode(self, x): h1 = F.relu(self.fc1(x)) h2 = self.fc2(h1) mean = h2[:, :self.z_size] logvar = h2[:, self.z_size:] return mean, logvar def sample_z(self, mu, logvar, k): B = mu.size()[0] eps = Variable( torch.FloatTensor(k, B, self.z_size).normal_().type( self.dtype)) #[P,B,Z] z = eps.mul(torch.exp(.5 * logvar)) + mu #[P,B,Z] logpz = lognormal( z, Variable(torch.zeros(B, self.z_size).type(self.dtype)), Variable(torch.zeros(B, self.z_size)).type(self.dtype)) #[P,B] logqz = lognormal(z, mu, logvar) return z, logpz, logqz def sample_W(self): Ws, log_p_W_sum, log_q_W_sum = self.decoder.sample_weights() return Ws, log_p_W_sum, log_q_W_sum def decode(self, Ws, z): k = z.size()[0] B = z.size()[1] z = z.view(-1, self.z_size) x = self.decoder.forward(Ws, z) x = x.view(k, B, self.input_size) return x def forward(self, x, k, s): self.B = x.size()[0] #batch size # self.k = k #number of z samples aka particles P # self.s = s #number of W samples elbo1s = [] logprobs = [[] for _ in range(5)] for i in range(s): Ws, logpW, logqW = self.sample_W() #_ , [1], [1] mu, logvar = self.encode(x) #[B,Z] z, logpz, logqz = self.sample_z(mu, logvar, k=k) #[P,B,Z], [P,B] x_hat = self.decode(Ws, z) #[P,B,X] logpx = log_bernoulli(x_hat, x) #[P,B] elbo = logpx + logpz - logqz #[P,B] if k > 1: max_ = torch.max(elbo, 0)[0] #[B] elbo1 = torch.log(torch.mean(torch.exp(elbo - max_), 0)) + max_ #[B] elbo = elbo + (logpW * .000001) - (logqW * self.qW_weight ) #[B], logp(x|W)p(w)/q(w) elbo1s.append(elbo) logprobs[0].append(torch.mean(logpx)) logprobs[1].append(torch.mean(logpz)) logprobs[2].append(torch.mean(logqz)) logprobs[3].append(torch.mean(logpW)) logprobs[4].append(torch.mean(logqW)) elbo1s = torch.stack(elbo1s) #[S,B] if s > 1: max_ = torch.max(elbo1s, 0)[0] #[B] elbo1 = torch.log(torch.mean(torch.exp(elbo1s - max_), 0)) + max_ #[B] elbo = torch.mean(elbo1s) #[1] #for printing # logpx = torch.mean(logpx) # logpz = torch.mean(logpz) # logqz = torch.mean(logqz) # self.x_hat_sigmoid = F.sigmoid(x_hat) logprobs2 = [torch.mean(torch.stack(aa)) for aa in logprobs] return elbo, logprobs2[0], logprobs2[1], logprobs2[2], logprobs2[ 3], logprobs2[4] def reconstruct(self, x): Ws, logpW, logqW = self.sample_W() #_ , [1], [1] mu, logvar = self.encode(x) #[B,Z] z, logpz, logqz = self.sample_z(mu, logvar, k=1) #[P,B,Z], [P,B] x_hat = self.decode(Ws, z) #[P,B,X] return F.sigmoid(x_hat) def predictive_elbo(self, x, k, s): # No pW or qW self.B = x.size()[0] #batch size # self.k = k #number of z samples aka particles P # self.s = s #number of W samples elbo1s = [] for i in range(s): Ws, logpW, logqW = self.sample_W() #_ , [1], [1] mu, logvar = self.encode(x) #[B,Z] z, logpz, logqz = self.sample_z(mu, logvar, k=k) #[P,B,Z], [P,B] x_hat = self.decode(Ws, z) #[P,B,X] logpx = log_bernoulli(x_hat, x) #[P,B] elbo = logpx + logpz - logqz #[P,B] if k > 1: max_ = torch.max(elbo, 0)[0] #[B] elbo = torch.log(torch.mean(torch.exp(elbo - max_), 0)) + max_ #[B] # elbo1 = elbo1 #+ (logpW - logqW)*.00000001 #[B], logp(x|W)p(w)/q(w) elbo1s.append(elbo) elbo1s = torch.stack(elbo1s) #[S,B] if s > 1: max_ = torch.max(elbo1s, 0)[0] #[B] elbo1 = torch.log(torch.mean(torch.exp(elbo1s - max_), 0)) + max_ #[B] elbo = torch.mean(elbo1s) #[1] return elbo #, logprobs2[0], logprobs2[1], logprobs2[2], logprobs2[3], logprobs2[4]
class BVAE(nn.Module): def __init__(self): super(BVAE, self).__init__() if torch.cuda.is_available(): self.dtype = torch.cuda.FloatTensor else: self.dtype = torch.FloatTensor self.z_size = 20 self.input_size = 784 #Encoder self.fc1 = nn.Linear(self.input_size, 200) self.fc2 = nn.Linear(200, self.z_size * 2) #Decoder # self.fc3 = nn.Linear(self.z_size, 200) # self.fc4 = nn.Linear(200, 784) # self.decoder = BNN([self.z_size, 200, 784], [torch.nn.Softplus, torch.nn.Softplus]) self.decoder = BNN([self.z_size, 200, 784], [F.relu, F.relu]) # self.add_module('BNN', self.decoder) # for idx, m in enumerate(self.modules()): # print(idx, '->', m) # fsdf def encode(self, x): h1 = F.relu(self.fc1(x)) h2 = self.fc2(h1) mean = h2[:, :self.z_size] logvar = h2[:, self.z_size:] return mean, logvar def sample(self, mu, logvar, k): # if torch.cuda.is_available(): # eps = Variable(torch.FloatTensor(k, self.B, self.z_size).normal_()).cuda() #[P,B,Z] # else: eps = Variable( torch.FloatTensor(k, self.B, self.z_size).normal_().type( self.dtype)) #[P,B,Z] z = eps.mul(torch.exp(.5 * logvar)) + mu #[P,B,Z] # if torch.cuda.is_available(): # logpz = lognormal(z, Variable(torch.zeros(self.B, self.z_size).cuda()), # Variable(torch.zeros(self.B, self.z_size)).cuda()) #[P,B] # else: logpz = lognormal( z, Variable(torch.zeros(self.B, self.z_size).type(self.dtype)), Variable(torch.zeros(self.B, self.z_size)).type(self.dtype)) #[P,B] logqz = lognormal(z, mu, logvar) return z, logpz, logqz def decode(self, z): # h3 = F.relu(self.fc3(z)) # return self.fc4(h3) z = z.view(-1, self.z_size) Ws, log_p_W_sum, log_q_W_sum = self.decoder.sample_weights() x = self.decoder.forward(Ws, z) x = x.view(self.k, self.B, self.input_size) return x, log_p_W_sum, log_q_W_sum def forward(self, x, k=1): self.k = k self.B = x.size()[0] mu, logvar = self.encode(x) z, logpz, logqz = self.sample(mu, logvar, k=k) x_hat, logpW, logqW = self.decode(z) logpx = log_bernoulli(x_hat, x) #[P,B] elbo = logpx + logpz - logqz + (logpW - logqW) * .00000001 #[P,B] if k > 1: max_ = torch.max(elbo, 0)[0] #[B] elbo = torch.log(torch.mean(torch.exp(elbo - max_), 0)) + max_ #[B] elbo = torch.mean(elbo) #[1] #for printing logpx = torch.mean(logpx) logpz = torch.mean(logpz) logqz = torch.mean(logqz) self.x_hat_sigmoid = F.sigmoid(x_hat) return elbo, logpx, logpz, logqz, logpW, logqW
class BVAE(nn.Module): def __init__(self, qW_weight, seed=-1): super(BVAE, self).__init__() if seed != -1: torch.manual_seed(seed) if torch.cuda.is_available(): self.dtype = torch.cuda.FloatTensor else: self.dtype = torch.FloatTensor self.qW_weight = qW_weight self.z_size = 2 self.input_size = 784 #Encoder self.fc1 = nn.Linear(self.input_size, 200) self.fc2 = nn.Linear(200, self.z_size*2) #Decoder self.decoder = BNN([self.z_size, 200, 784], [F.relu, F.relu]) def encode(self, x): h1 = F.relu(self.fc1(x)) h2 = self.fc2(h1) mean = h2[:,:self.z_size] logvar = h2[:,self.z_size:] return mean, logvar def sample_z(self, mu, logvar, k): B = mu.size()[0] eps = Variable(torch.FloatTensor(k, B, self.z_size).normal_().type(self.dtype)) #[P,B,Z] z = eps.mul(torch.exp(.5*logvar)) + mu #[P,B,Z] logpz = lognormal(z, Variable(torch.zeros(B, self.z_size).type(self.dtype)), Variable(torch.zeros(B, self.z_size)).type(self.dtype)) #[P,B] logqz = lognormal(z, mu, logvar) return z, logpz, logqz def sample_W(self): Ws, log_p_W_sum, log_q_W_sum = self.decoder.sample_weights() return Ws, log_p_W_sum, log_q_W_sum def decode(self, Ws, z): k = z.size()[0] B = z.size()[1] z = z.view(-1, self.z_size) x = self.decoder.forward(Ws, z) x = x.view(k, B, self.input_size) return x def forward(self, x, k, s): self.B = x.size()[0] #batch size # self.k = k #number of z samples aka particles P # self.s = s #number of W samples elbo1s = [] logprobs = [[] for _ in range(5)] for i in range(s): Ws, logpW, logqW = self.sample_W() #_ , [1], [1] mu, logvar = self.encode(x) #[B,Z] z, logpz, logqz = self.sample_z(mu, logvar, k=k) #[P,B,Z], [P,B] x_hat = self.decode(Ws, z) #[P,B,X] logpx = log_bernoulli(x_hat, x) #[P,B] elbo = logpx + logpz - logqz #[P,B] if k>1: max_ = torch.max(elbo, 0)[0] #[B] elbo1 = torch.log(torch.mean(torch.exp(elbo - max_), 0)) + max_ #[B] elbo = elbo + (logpW*.000001) - (logqW*self.qW_weight) #[B], logp(x|W)p(w)/q(w) elbo1s.append(elbo) logprobs[0].append(torch.mean(logpx)) logprobs[1].append(torch.mean(logpz)) logprobs[2].append(torch.mean(logqz)) logprobs[3].append(torch.mean(logpW)) logprobs[4].append(torch.mean(logqW)) elbo1s = torch.stack(elbo1s) #[S,B] if s>1: max_ = torch.max(elbo1s, 0)[0] #[B] elbo1 = torch.log(torch.mean(torch.exp(elbo1s - max_), 0)) + max_ #[B] elbo = torch.mean(elbo1s) #[1] #for printing # logpx = torch.mean(logpx) # logpz = torch.mean(logpz) # logqz = torch.mean(logqz) # self.x_hat_sigmoid = F.sigmoid(x_hat) logprobs2 = [torch.mean(torch.stack(aa)) for aa in logprobs] return elbo, logprobs2[0], logprobs2[1], logprobs2[2], logprobs2[3], logprobs2[4] def reconstruct(self, x): Ws, logpW, logqW = self.sample_W() #_ , [1], [1] mu, logvar = self.encode(x) #[B,Z] z, logpz, logqz = self.sample_z(mu, logvar, k=1) #[P,B,Z], [P,B] x_hat = self.decode(Ws, z) #[P,B,X] return F.sigmoid(x_hat) def predictive_elbo(self, x, k, s): # No pW or qW self.B = x.size()[0] #batch size # self.k = k #number of z samples aka particles P # self.s = s #number of W samples elbo1s = [] for i in range(s): Ws, logpW, logqW = self.sample_W() #_ , [1], [1] mu, logvar = self.encode(x) #[B,Z] z, logpz, logqz = self.sample_z(mu, logvar, k=k) #[P,B,Z], [P,B] x_hat = self.decode(Ws, z) #[P,B,X] logpx = log_bernoulli(x_hat, x) #[P,B] elbo = logpx + logpz - logqz #[P,B] if k>1: max_ = torch.max(elbo, 0)[0] #[B] elbo = torch.log(torch.mean(torch.exp(elbo - max_), 0)) + max_ #[B] # elbo1 = elbo1 #+ (logpW - logqW)*.00000001 #[B], logp(x|W)p(w)/q(w) elbo1s.append(elbo) elbo1s = torch.stack(elbo1s) #[S,B] if s>1: max_ = torch.max(elbo1s, 0)[0] #[B] elbo1 = torch.log(torch.mean(torch.exp(elbo1s - max_), 0)) + max_ #[B] elbo = torch.mean(elbo1s) #[1] return elbo#, logprobs2[0], logprobs2[1], logprobs2[2], logprobs2[3], logprobs2[4]
class BVAE(nn.Module): def __init__(self): super(BVAE, self).__init__() if torch.cuda.is_available(): self.dtype = torch.cuda.FloatTensor else: self.dtype = torch.FloatTensor self.z_size = 20 self.input_size = 784 #Encoder self.fc1 = nn.Linear(self.input_size, 200) self.fc2 = nn.Linear(200, self.z_size*2) #Decoder # self.fc3 = nn.Linear(self.z_size, 200) # self.fc4 = nn.Linear(200, 784) # self.decoder = BNN([self.z_size, 200, 784], [torch.nn.Softplus, torch.nn.Softplus]) self.decoder = BNN([self.z_size, 200, 784], [F.relu, F.relu]) # self.add_module('BNN', self.decoder) # for idx, m in enumerate(self.modules()): # print(idx, '->', m) # fsdf def encode(self, x): h1 = F.relu(self.fc1(x)) h2 = self.fc2(h1) mean = h2[:,:self.z_size] logvar = h2[:,self.z_size:] return mean, logvar def sample(self, mu, logvar, k): # if torch.cuda.is_available(): # eps = Variable(torch.FloatTensor(k, self.B, self.z_size).normal_()).cuda() #[P,B,Z] # else: eps = Variable(torch.FloatTensor(k, self.B, self.z_size).normal_().type(self.dtype)) #[P,B,Z] z = eps.mul(torch.exp(.5*logvar)) + mu #[P,B,Z] # if torch.cuda.is_available(): # logpz = lognormal(z, Variable(torch.zeros(self.B, self.z_size).cuda()), # Variable(torch.zeros(self.B, self.z_size)).cuda()) #[P,B] # else: logpz = lognormal(z, Variable(torch.zeros(self.B, self.z_size).type(self.dtype)), Variable(torch.zeros(self.B, self.z_size)).type(self.dtype)) #[P,B] logqz = lognormal(z, mu, logvar) return z, logpz, logqz def decode(self, z): # h3 = F.relu(self.fc3(z)) # return self.fc4(h3) z = z.view(-1, self.z_size) Ws, log_p_W_sum, log_q_W_sum = self.decoder.sample_weights() x = self.decoder.forward(Ws, z) x = x.view(self.k, self.B, self.input_size) return x, log_p_W_sum, log_q_W_sum def forward(self, x, k=1): self.k = k self.B = x.size()[0] mu, logvar = self.encode(x) z, logpz, logqz = self.sample(mu, logvar, k=k) x_hat, logpW, logqW = self.decode(z) logpx = log_bernoulli(x_hat, x) #[P,B] elbo = logpx + logpz - logqz + (logpW - logqW)*.00000001 #[P,B] if k>1: max_ = torch.max(elbo, 0)[0] #[B] elbo = torch.log(torch.mean(torch.exp(elbo - max_), 0)) + max_ #[B] elbo = torch.mean(elbo) #[1] #for printing logpx = torch.mean(logpx) logpz = torch.mean(logpz) logqz = torch.mean(logqz) self.x_hat_sigmoid = F.sigmoid(x_hat) return elbo, logpx, logpz, logqz, logpW, logqW