def forward3(self, x, k=1):
self.B = x.size()[0]
mu, logvar = self.encode(x)
z, logpz, logqz = self.sample(mu, logvar, k=k) #[P,B,Z]
x_hat = self.decode(z) #[PB,X]
x_hat = x_hat.view(k, self.B, -1)
# print x_hat.size()
x = x.view(self.B, -1)
# print (x_hat.size())
# print (x.size())
# fasdfd
logpx = log_bernoulli(x_hat, x) #[P,B]
elbo = logpx #+ logpz - logqz #[P,B]
if k>1:
max_ = torch.max(elbo, 0)[0] #[B]
elbo = torch.log(torch.mean(torch.exp(elbo - max_), 0)) + max_ #[B]
elbo = torch.mean(elbo) #[1]
#for printing
logpx = torch.mean(logpx)
logpz = torch.mean(logpz)
logqz = torch.mean(logqz)
self.x_hat_sigmoid = F.sigmoid(x_hat)
return elbo, logpx, logpz, logqz, self.x_hat_sigmoid
def forward(self, x, policy, k=1):
# x: [B,2,84,84]
self.B = x.size()[0]
mu, logvar = self.encode(x)
# print (mu.size())
# print (logvar.size())
# mu = mu.unsqueeze(0)
# logvar = logvar.unsqueeze(0)
z, logpz, logqz = self.sample(mu, logvar, k=k) #[P,B,Z]
x_hat = self.decode(z) #[PB,X]
x_hat_sigmoid = F.sigmoid(x_hat)
# print (torch.FloatTensor(x_hat_sigmoid.size()).random_(0,10) )
# print (torch.FloatTensor(x_hat_sigmoid.size()).uniform_(0,.3) )
# fsad
#Add Noise
# noisy_x_hat_sigmoid = x_hat_sigmoid + Variable(torch.FloatTensor(x_hat_sigmoid.size()).uniform_(0,.3)).cuda()
# x_hat_sigmoid = noisy_x_hat_sigmoid
# dist_recon = policy.action_dist(F.sigmoid(x_hat)*255.)
# log_dist_recon = policy.action_logdist(F.sigmoid(x_hat)*255.)
log_dist_recon = policy.action_logdist(x_hat_sigmoid)
# print (torch.sum(torch.autograd.grad(torch.sum(torch.log(dist_recon)), self.deconv3.weight)[0]))
# print (torch.sum(torch.autograd.grad(torch.sum(torch.log(dist_recon)), self.deconv3.weight)[0]))
# print (torch.sum(torch.autograd.grad(torch.sum(x_hat*10), self.deconv3.weight)[0]))
# fsadf
# dist_true = policy.action_dist(x*255.)
# log_dist_true = policy.action_logdist(x*255.)
log_dist_true = policy.action_logdist(x)
# print (dist_true)
# print (dist_recon)
# fasdf
# # print (x_hat.size())
# # print (dist_recon)
# # print (dist_true)
# # fads
flat_x_hat = x_hat.view(k, self.B, -1)
# # print x_hat.size()
flat_x = x.view(self.B, -1)
# # print (x_hat.size())
# # print (x.size())
# # fasdfd
logpx = log_bernoulli(flat_x_hat, flat_x) #[P,B]
# print (logpx.size())
# elbo = logpx + logpz - logqz #[P,B]
# action_dif = torch.mean((dist_recon-dist_true)**2)
# action_dif = torch.sum((torch.log(dist_true) - torch.log(dist_recon))*dist_true)
action_dist_kl = torch.sum((log_dist_true - log_dist_recon)*torch.exp(log_dist_true), dim=1) #[B]
# print (action_dif.size())
# fasdf
# neg_action_dif = - action_dif
# print (torch.sum(torch.autograd.grad(neg_action_dif, policy.conv1.weight)[0])) # ZERO
# fdsa
# print (torch.sum(torch.autograd.grad(neg_action_dif, self.deconv3.weight)[0]))
# print (torch.sum(torch.autograd.grad(neg_action_dif, self.deconv1.weight)[0]))
# print (torch.sum(torch.autograd.grad(neg_action_dif, self.conv1.weight)[0]))
# print (torch.sum(torch.autograd.grad(torch.sum(torch.log(dist_recon)), self.deconv3.weight)[0]))
# print (torch.sum(torch.autograd.grad(torch.sum(torch.log(dist_true)), self.deconv3.weight)[0]))
# fadf
# elbo = logpx + logpz - logqz
# if k>1:
# max_ = torch.max(elbo, 0)[0] #[B]
# elbo = torch.log(torch.mean(torch.exp(elbo - max_), 0)) + max_ #[B]
# elbo = torch.mean(elbo) #[1]
# neg_action_dif = neg_action_dif
# logpz = logpz*.01
# logqz = logqz*.01
# logpx = logpx*.01
# elbo = torch.mean(logpx) + torch.mean(logpz) - torch.mean(logqz) - torch.mean(action_dist_kl) #[1]
#for printing
action_dist_kl = torch.mean(action_dist_kl)
weight = .01 # .00001
logpx = torch.mean(logpx) * weight * 0.
logpz = torch.mean(logpz) * weight
logqz = torch.mean(logqz) * weight
elbo = torch.mean(logpx) + torch.mean(logpz) - torch.mean(logqz) - torch.mean(action_dist_kl) #[1]
# self.x_hat_sigmoid = F.sigmoid(x_hat)
# return elbo, logpx, logpz, logqz
# print (logpx != logpx)
# if (logpx != logpx).data.cpu().numpy():
# print( 'NAN')
# fasd
return elbo, logpx, logpz, logqz, action_dist_kl
def forward(self, x, policies, k=1):
# x: [B,2,84,84]
self.B = x.size()[0]
mu, logvar = self.encode(x)
z, logpz, logqz = self.sample(mu, logvar, k=k) #[P,B,Z]
x_hat = self.decode(z) #[PB,X]
x_hat_sigmoid = F.sigmoid(x_hat)
kls = []
act_difs = []
for p in range(len(policies)):
log_dist_recon = policies[p].action_logdist(x_hat_sigmoid)
log_dist_true = policies[p].action_logdist(x)
action_dist_kl = torch.sum((log_dist_true - log_dist_recon)*torch.exp(log_dist_true), dim=1) #[B]
kls.append(action_dist_kl)
act_recon = policies[p].get_intermediate_activation(x_hat_sigmoid)
act_recon = act_recon.view(self.B, -1)
act_true = policies[p].get_intermediate_activation(x)
act_true = act_true.view(self.B, -1)
# print (act_true)
# fsadf
act_dif = torch.mean((act_recon - act_true)**2, dim=1)
act_difs.append(act_dif)
#Average over polices
kls = torch.stack(kls) #[policies, B]
action_dist_kl = torch.mean(kls, dim=0) #[B]
action_dist_kl = torch.mean(action_dist_kl) #[1]
act_difs = torch.stack(act_difs) #[policies, B]
act_dif = torch.mean(act_dif, dim=0) #[B]
act_dif = torch.mean(act_dif) #[1]
#Likelihood
flat_x_hat = x_hat.view(k, self.B, -1)
flat_x = x.view(self.B, -1)
logpx = log_bernoulli(flat_x_hat, flat_x) #[P,B]
# scale = action_dist_kl.data / (torch.mean(logpx) + torch.mean(logpz) - torch.mean(logqz)).data
# scale = torch.clamp(scale, max=1.)
# scale = Variable(scale)
scale = .00001
logpx = torch.mean(logpx) * scale #* 0.1
logpz = torch.mean(logpz) * scale
logqz = torch.mean(logqz) * scale
elbo = torch.mean(logpx) + torch.mean(logpz) - torch.mean(logqz) - torch.mean(action_dist_kl) - act_dif #[1]
return elbo, logpx, logpz, logqz, action_dist_kl, act_dif
def forward(self, x, policy, k=1):
# x: [B,2,84,84]
self.B = x.size()[0]
mu, logvar = self.encode(x)
# print (mu.size())
# print (logvar.size())
# mu = mu.unsqueeze(0)
# logvar = logvar.unsqueeze(0)
z, logpz, logqz = self.sample(mu, logvar, k=k) #[P,B,Z]
x_hat = self.decode(z) #[PB,X]
x_hat_sigmoid = F.sigmoid(x_hat)
# print (torch.FloatTensor(x_hat_sigmoid.size()).random_(0,10) )
# print (torch.FloatTensor(x_hat_sigmoid.size()).uniform_(0,.3) )
# fsad
#Add Noise
# noisy_x_hat_sigmoid = x_hat_sigmoid + Variable(torch.FloatTensor(x_hat_sigmoid.size()).uniform_(0,.3)).cuda()
# x_hat_sigmoid = noisy_x_hat_sigmoid
# dist_recon = policy.action_dist(F.sigmoid(x_hat)*255.)
# log_dist_recon = policy.action_logdist(F.sigmoid(x_hat)*255.)
log_dist_recon = policy.action_logdist(x_hat_sigmoid)
# print (torch.sum(torch.autograd.grad(torch.sum(torch.log(dist_recon)), self.deconv3.weight)[0]))
# print (torch.sum(torch.autograd.grad(torch.sum(torch.log(dist_recon)), self.deconv3.weight)[0]))
# print (torch.sum(torch.autograd.grad(torch.sum(x_hat*10), self.deconv3.weight)[0]))
# fsadf
# dist_true = policy.action_dist(x*255.)
# log_dist_true = policy.action_logdist(x*255.)
log_dist_true = policy.action_logdist(x)
# print (dist_true)
# print (dist_recon)
# fasdf
# # print (x_hat.size())
# # print (dist_recon)
# # print (dist_true)
# # fads
flat_x_hat = x_hat.view(k, self.B, -1)
# # print x_hat.size()
flat_x = x.view(self.B, -1)
# # print (x_hat.size())
# # print (x.size())
# # fasdfd
logpx = log_bernoulli(flat_x_hat, flat_x) #[P,B]
# print (logpx.size())
# elbo = logpx + logpz - logqz #[P,B]
# action_dif = torch.mean((dist_recon-dist_true)**2)
# action_dif = torch.sum((torch.log(dist_true) - torch.log(dist_recon))*dist_true)
action_dist_kl = torch.sum((log_dist_true - log_dist_recon)*torch.exp(log_dist_true), dim=1) #[B]
# print (action_dif.size())
# fasdf
# neg_action_dif = - action_dif
# print (torch.sum(torch.autograd.grad(neg_action_dif, policy.conv1.weight)[0])) # ZERO
# fdsa
# print (torch.sum(torch.autograd.grad(neg_action_dif, self.deconv3.weight)[0]))
# print (torch.sum(torch.autograd.grad(neg_action_dif, self.deconv1.weight)[0]))
# print (torch.sum(torch.autograd.grad(neg_action_dif, self.conv1.weight)[0]))
# print (torch.sum(torch.autograd.grad(torch.sum(torch.log(dist_recon)), self.deconv3.weight)[0]))
# print (torch.sum(torch.autograd.grad(torch.sum(torch.log(dist_true)), self.deconv3.weight)[0]))
# fadf
# elbo = logpx + logpz - logqz
# if k>1:
# max_ = torch.max(elbo, 0)[0] #[B]
# elbo = torch.log(torch.mean(torch.exp(elbo - max_), 0)) + max_ #[B]
# elbo = torch.mean(elbo) #[1]
# neg_action_dif = neg_action_dif
# logpz = logpz*.01
# logqz = logqz*.01
# logpx = logpx*.01
# elbo = torch.mean(logpx) + torch.mean(logpz) - torch.mean(logqz) - torch.mean(action_dist_kl) #[1]
#for printing
action_dist_kl = torch.mean(action_dist_kl)
weight = .01 # .00001
logpx = torch.mean(logpx) * weight * 0.
logpz = torch.mean(logpz) * weight
logqz = torch.mean(logqz) * weight
elbo = torch.mean(logpx) + torch.mean(logpz) - torch.mean(logqz) - torch.mean(action_dist_kl) #[1]
# self.x_hat_sigmoid = F.sigmoid(x_hat)
# return elbo, logpx, logpz, logqz
# print (logpx != logpx)
# if (logpx != logpx).data.cpu().numpy():
# print( 'NAN')
# fasd
return elbo, logpx, logpz, logqz, action_dist_kl
def forward(self, x, policies, k=1):
# x: [B,2,84,84]
self.B = x.size()[0]
mu, logvar = self.encode(x)
# z, logpz, logqz = self.sample(mu, logvar, k=k) #[P,B,Z]
# x_hat = self.decode(z) #[PB,X]
x_hat = self.decode(mu) #[PB,X]
x_hat_sigmoid = F.sigmoid(x_hat)
kls = []
act_difs = []
grad_difs = []
for p in range(len(policies)):
x = Variable(x.data, requires_grad=True, volatile=False)
log_dist_recon = policies[p].action_logdist(x_hat_sigmoid)
log_dist_true = policies[p].action_logdist(x)
action_dist_kl = torch.sum(
(log_dist_true - log_dist_recon) * torch.exp(log_dist_true),
dim=1) #[B]
kls.append(action_dist_kl)
# act_recon = policies[p].get_intermediate_activation(x_hat_sigmoid)
# act_recon = act_recon.view(self.B, -1)
# act_true = policies[p].get_intermediate_activation(x)
# act_true = act_true.view(self.B, -1)
# # print (act_true)
# # fsadf
# act_dif = torch.mean((act_recon - act_true)**2, dim=1)
# act_difs.append(act_dif)
# ent_true = torch.mean(torch.sum(log_dist_true*torch.exp(log_dist_true),dim=1))
ent_true = torch.mean(log_dist_true[:, 3])
grad_true = torch.autograd.grad(ent_true,
x,
create_graph=True,
retain_graph=True)[0] #[B,2,84,84]
# print (grad_true)
# ent_recon = torch.mean(torch.sum(log_dist_recon*torch.exp(log_dist_recon),dim=1))
ent_recon = torch.mean(log_dist_recon[:, 3])
grad_recon = torch.autograd.grad(
ent_recon, x_hat_sigmoid, create_graph=True,
retain_graph=True)[0] #[B,2,84,84]
# print (grad_recon)
# fasd
# grad_dif = torch.mean((grad_recon-grad_true)**2) #[1]
grad_dif = torch.sum((grad_recon - grad_true)**2) #[1]
grad_difs.append(grad_dif)
# #Average over polices
kls = torch.stack(kls) #[policies, B]
# action_dist_kl = torch.mean(kls, dim=0) #[B]
action_dist_kl = torch.mean(action_dist_kl) #[1]
# act_difs = torch.stack(act_difs) #[policies, B]
# act_dif = torch.mean(act_dif, dim=0) #[B]
# act_dif = torch.mean(act_dif) #[1]
grad_difs = torch.stack(grad_difs) #[policies, B]
grad_dif = torch.mean(grad_difs) #*100. #[1]
# grad_dif = torch.sum(grad_difs) #*100. #[1]
#Likelihood
flat_x_hat = x_hat.view(k, self.B, -1)
flat_x = x.view(self.B, -1)
logpx = log_bernoulli(flat_x_hat, flat_x) #[P,B]
# scale = action_dist_kl.data / (torch.mean(logpx) + torch.mean(logpz) - torch.mean(logqz)).data
# scale = torch.clamp(scale, max=1.)
# scale = Variable(scale)
# scale = .00001
logpx = torch.mean(logpx) #* scale #* 0.1
# logpz = torch.mean(logpz) * scale
# logqz = torch.mean(logqz) * scale
# elbo = torch.mean(logpx) + torch.mean(logpz) - torch.mean(logqz) - torch.mean(action_dist_kl) - act_dif - grad_dif #[1]
loss = grad_dif + action_dist_kl #[1]
# return elbo, logpx, logpz, logqz, action_dist_kl, act_dif, grad_dif
return loss, logpx, grad_dif, action_dist_kl