def posterior(self, x, ctx, use_mean=False): """ Encode the posterior. """ dists = [] out = x for layer_idx, layer in enumerate(self.posterior_net): if isinstance(layer, layers.ConvLSTM): # Process Condition cur_ctx = ctx.view(ctx.shape[0], -1, ctx.shape[-2], ctx.shape[-1]).unsqueeze(1) cur_ctx = self.posterior_init_net(cur_ctx) cur_ctx = cur_ctx.squeeze(1) # Run LSTM with the inputs from the previous layer out = layer(out, torch.chunk(cur_ctx, 2, 1)) else: out = layer(out) # Tanh the activations out = torch.tanh(out) # Compute distribution stats mean, logvar = torch.chunk(out, 2, 2) # Generate sample from this distribution z0 = flows.gaussian_rsample(mean, logvar, use_mean=use_mean) dists.append([mean, logvar, z0, z0, None]) return dists
def prior(self, emb, q_dists, use_mean=False, scale_var=1.): dists = [] for net_idx in range(len(self.prior_nets)): # print('[NET] PriorNet {}'.format(net_idx)) # Find the corresopnding activations ctx_idx = self.arch['latent']['ctx_idx'][net_idx] out = emb[ctx_idx][:, self.n_ctx - 1:-1].contiguous() cur_ctx = emb[ctx_idx][:, :self.n_ctx].contiguous() branch_layers = self.prior_nets[net_idx] # Process the current branch for branch_layer_idx, layer in enumerate(branch_layers): # print('[NET] PriorNet {}/{}'.format(net_idx, branch_layer_idx)) if isinstance(layer, ConvLSTM): # Get initial condition cur_ctx = cur_ctx.view(cur_ctx.shape[0], -1, cur_ctx.shape[-2], cur_ctx.shape[-1]) cur_ctx = cur_ctx.unsqueeze(1) cur_ctx = self.prior_init_nets[net_idx](cur_ctx) cur_ctx = cur_ctx.squeeze(1) # Forward LSTM out = layer(out, torch.chunk(cur_ctx, 2, 1)) else: out = layer(out) # Compute distribution stats mean, var = torch.chunk(out, 2, 2) # Scale the variance var = var * scale_var # Softplus var logvar = F.softplus(var).log() # Generate sample from this distribution z0 = flows.gaussian_rsample(mean, logvar, use_mean=use_mean) dists.append([mean, logvar, z0, z0, None]) return dists
def posterior(self, x, ctx, use_mean=False): dists = [] sto_branches = sorted(self.sto_branches.keys(), reverse=True) for layer_idx in sto_branches: # print(layer_idx) # Find the corresopnding activations out = x[layer_idx][:, self.n_ctx:].contiguous() cur_ctx = ctx[layer_idx][:, :self.n_ctx].contiguous() branch_layers = self.posterior_branches['layer_{}'.format( layer_idx)] # Process the current branch for branch_layer_idx, layer in enumerate(branch_layers): # print(branch_layer_idx) if isinstance(layer, layers.ConvLSTM): # Get initial condition cur_ctx = cur_ctx.view(cur_ctx.shape[0], -1, cur_ctx.shape[-2], cur_ctx.shape[-1]) cur_ctx = cur_ctx.unsqueeze(1) cur_ctx = self.posterior_init_nets['layer_{}'.format( layer_idx)](cur_ctx) cur_ctx = cur_ctx.squeeze(1) # Forward LSTM out = layer(out, torch.chunk(cur_ctx, 2, 1)) else: out = layer(out) # Compute distribution stats mean, var = torch.chunk(out, 2, 2) # Softplus var logvar = F.softplus(var).log() # Generate sample from this distribution z0 = flows.gaussian_rsample(mean, logvar, use_mean=use_mean) dists.append([mean, logvar, z0, z0, None]) return dists
def posterior(self, x, ctx, use_mean=False, scale_var=1.): dists = [] sto_branches = sorted(self.sto_branches.keys(), reverse=True) for layer_idx in sto_branches: # print(layer_idx) # Find the corresopnding activations out = x[layer_idx][:, self.n_ctx:].contiguous() cur_ctx = ctx[layer_idx][:, :self.n_ctx].contiguous() branch_layers = self.posterior_branches['layer_{}'.format(layer_idx)] # Process the current branch for branch_layer_idx, layer in enumerate(branch_layers): # print(branch_layer_idx) if isinstance(layer, layers.ConvLSTM): # Get initial condition cur_ctx = cur_ctx.view(cur_ctx.shape[0], -1, cur_ctx.shape[-2], cur_ctx.shape[-1]) cur_ctx = cur_ctx.unsqueeze(1) cur_ctx = self.posterior_init_nets['layer_{}'.format(layer_idx)](cur_ctx) cur_ctx = cur_ctx.squeeze(1) # Forward LSTM out = layer(out, torch.chunk(cur_ctx, 2, 1)) # Handcrafted rules for integrating the different z's elif branch_layer_idx == 3: if layer_idx == 16: out = layer(out) elif layer_idx == 10: z1 = dists[0][-2] b, t, c, h, w = z1.shape z1 = z1.view(b*t, c, h, w) z1 = F.interpolate(z1, scale_factor=8) z1 = z1.view(b, t, c, z1.shape[-2], z1.shape[-1]) out = torch.cat([out, z1], 2) out = layer(out) elif layer_idx == 4: z1 = dists[0][-2] z2 = dists[1][-2] b, t, c, h, w = z1.shape z1 = z1.view(b*t, c, h, w) z1 = F.interpolate(z1, scale_factor=32) z1 = z1.view(b, t, c, z1.shape[-2], z1.shape[-1]) b, t, c, h, w = z2.shape z2 = z2.view(b*t, c, h, w) z2 = F.interpolate(z2, scale_factor=4) z2 = z2.view(b, t, c, z2.shape[-2], z2.shape[-1]) out = torch.cat([out, z1, z2], 2) out = layer(out) else: out = layer(out) # Compute distribution stats mean, var = torch.chunk(out, 2, 2) # Softplus var scaled_var = F.softplus(var)*scale_var logvar = scaled_var.log() # Generate sample from this distribution z0 = flows.gaussian_rsample(mean, logvar, use_mean=use_mean) dists.append([mean, logvar, z0, z0, None]) return dists
def posterior(self, emb, use_mean=False, scale_var=1.): dists = [] for net_idx in range(len(self.posterior_nets)): # print('[NET] PosteriorNet {}'.format(net_idx)) # Find the corresopnding activations ctx_idx = self.arch['latent']['ctx_idx'][net_idx] out = emb[ctx_idx][:, self.n_ctx:].contiguous() cur_ctx = emb[ctx_idx][:, :self.n_ctx].contiguous() branch_layers = self.posterior_nets[net_idx] # print('CTX IDX: ', ctx_idx, ' shape: ', cur_ctx.shape) # print(branch_layers) # Process the current branch for branch_layer_idx, layer in enumerate(branch_layers): # print('[NET] PosteriorNet {}/{}'.format(net_idx, branch_layer_idx)) if isinstance(layer, ConvLSTM): # Get initial condition cur_ctx = cur_ctx.view(cur_ctx.shape[0], -1, cur_ctx.shape[-2], cur_ctx.shape[-1]) cur_ctx = cur_ctx.unsqueeze(1) cur_ctx = self.posterior_init_nets[net_idx](cur_ctx) cur_ctx = cur_ctx.squeeze(1) # Forward LSTM out = layer(out, torch.chunk(cur_ctx, 2, 1)) # Dense connectivity latent elif branch_layer_idx == 3: # print('THIRD LAYER') # Get current latent resolution cur_res = self.arch['latent']['resolution'][net_idx] # Accumulate previous z prev_zs = [] for prev_z_idx in range(net_idx): # Get previous z resolution prev_res = self.arch['latent']['resolution'][ prev_z_idx] # Compute scaling factor scaling_factor = cur_res // prev_res # Interpolate previous z z_prev = dists[prev_z_idx][-2] b, t, c, h, w = z_prev.shape z_prev = z_prev.view(b * t, c, h, w) z_prev = F.interpolate(z_prev, scale_factor=scaling_factor) z_prev = z_prev.view(b, t, c, z_prev.shape[-2], z_prev.shape[-1]) prev_zs.append(z_prev) # Concatenate zs prev_zs = torch.cat(prev_zs + [out], 2) # Forward through layer out = layer(prev_zs) else: out = layer(out) # Compute distribution stats mean, var = torch.chunk(out, 2, 2) # Scale the variance var = var * scale_var # Softplus var logvar = F.softplus(var).log() # Generate sample from this distribution z0 = flows.gaussian_rsample(mean, logvar, use_mean=use_mean) dists.append([mean, logvar, z0, z0, None]) return dists