def preprocess(x):
"""Preprocess 98-dimensional heatmaps."""
N, C, H, W = x.size()
x = truncate(x)
x = normalize(x)
sw = H // 256
operations = Munch(chin=OPPAIR(0, 3),
eyebrows=OPPAIR(-7 * sw, 2),
nostrils=OPPAIR(8 * sw, 4),
lipupper=OPPAIR(-8 * sw, 4),
liplower=OPPAIR(8 * sw, 4),
lipinner=OPPAIR(-2 * sw, 3))
for part, ops in operations.items():
start, end = index_map[part]
x[:, start:end] = resize(shift(x[:, start:end], ops.shift), ops.resize)
zero_out = porch.cat([
porch.arange(0, index_map.chin.start),
porch.arange(index_map.chin.end, 33),
porch.LongTensor([
index_map.eyebrowsedges.start, index_map.eyebrowsedges.end,
index_map.lipedges.start, index_map.lipedges.end
])
])
x[:, zero_out] = 0
start, end = index_map.nose
x[:, start + 1:end] = shift(x[:, start + 1:end], 4 * sw)
x[:, start:end] = resize(x[:, start:end], 1)
start, end = index_map.eyes
x[:, start:end] = resize(x[:, start:end], 1)
x[:, start:end] = resize(shift(x[:, start:end], -8), 3) + \
shift(x[:, start:end], -24)
# Second-level mask
x2 = deepcopy(x)
x2[:, index_map.chin.start:index_map.chin.end] = 0 # start:end was 0:33
x2[:, index_map.lipedges.start:index_map.lipinner.
end] = 0 # start:end was 76:96
x2[:, index_map.eyebrows.start:index_map.eyebrows.
end] = 0 # start:end was 33:51
x = porch.sum(x, dim=1, keepdim=True) # (N, 1, H, W)
x2 = porch.sum(x2, dim=1, keepdim=True) # mask without faceline and mouth
x[x != x] = 0 # set nan to zero
x2[x != x] = 0 # set nan to zero
return x.clamp_(0, 1), x2.clamp_(0, 1)
def forward(self, x, y=None):
# Run input conv
h = self.input_conv(x)
# Loop over blocks
for index, blocklist in enumerate(self.blocks):
for block in blocklist:
h = block(h)
# Apply global sum pooling as in SN-GAN
h = torch.sum(self.activation(h), [2, 3])
# Get initial class-unconditional output
out = self.linear(h)
# Get projection of final featureset onto class vectors and add to evidence
out = out + torch.sum(self.embed(y) * h, 1, keepdim=True)
return out
def forward(self, inputs):
x = self.down_layer(inputs)
gap = porch.nn.functional.adaptive_avg_pool2d(x, 1)
gap_logit = self.gap_fc(gap.view(x.shape[0], -1))
gap_weight = porch.Tensor(list(self.gap_fc.parameters())[0]).permute(1,0)
gap = x * gap_weight.unsqueeze(2).unsqueeze(3)
gmp = porch.nn.functional.adaptive_max_pool2d(x, 1)
gmp_logit = self.gmp_fc(gmp.view(x.shape[0], -1))
gmp_weight = porch.Tensor(list(self.gmp_fc.parameters())[0]).permute(1,0)
gmp = x * gmp_weight.unsqueeze(2).unsqueeze(3)
cam_logit = porch.cat([gap_logit, gmp_logit], 1)
x = porch.cat([gap, gmp], 1)
x = self.relu(self.conv1x1(x))
x =porch.Tensor(x)
heatmap = porch.sum(x, dim=1, keepdim=True)
if self.light:
x_ = porch.nn.functional.adaptive_avg_pool2d(x, 1)
x_ = self.fc(x_.view(x_.shape[0], -1))
else:
x_ = self.fc(x.view(x.shape[0], -1))
gamma, beta = self.gamma(x_), self.beta(x_)
for i in range(self.n_res):
x = getattr(self, "ResNetAdaILNBlock_" + str(i + 1))(x, gamma, beta)
out = self.up_layer(x)
return out, cam_logit ,heatmap
def forward(self, input):
x = self.DownBlock(input)
gap = torch.nn.functional.adaptive_avg_pool2d(x, 1)
gap_logit = self.gap_fc(gap.view(x.shape[0], -1))
gap_weight = list(self.gap_fc.parameters())[0]
gap = x * gap_weight.unsqueeze(2).unsqueeze(3)
gmp = torch.nn.functional.adaptive_max_pool2d(x, 1)
gmp_logit = self.gmp_fc(gmp.view(x.shape[0], -1))
gmp_weight = list(self.gmp_fc.parameters())[0]
gmp = x * gmp_weight.unsqueeze(2).unsqueeze(3)
cam_logit = torch.cat([gap_logit, gmp_logit], 1)
x = torch.cat([gap, gmp], 1)
x = self.relu(self.conv1x1(x))
heatmap = torch.sum(x, dim=1, keepdim=True)
if self.light:
x_ = torch.nn.functional.adaptive_avg_pool2d(x, 1)
x_ = self.FC(x_.view(x_.shape[0], -1))
else:
x_ = self.FC(x.view(x.shape[0], -1))
gamma, beta = self.gamma(x_), self.beta(x_)
for i in range(self.n_blocks):
x = getattr(self, 'UpBlock1_' + str(i + 1))(x, gamma, beta)
out = self.UpBlock2(x)
return out, cam_logit, heatmap
def compute_d_loss(nets,
args,
x_real,
y_org,
y_trg,
z_trg=None,
x_ref=None,
masks=None):
assert (z_trg is None) != (x_ref is None)
# with real images
x_real.stop_gradient = False
out = nets.discriminator(x_real, y_org)
loss_real = adv_loss(out, 1)
loss_reg = r1_reg(out, x_real)
# with fake images
with porch.no_grad():
if z_trg is not None:
s_trg = nets.mapping_network(z_trg, y_trg)
else: # x_ref is not None
s_trg = nets.style_encoder(x_ref, y_trg)
x_fake = nets.generator(x_real, s_trg, masks=masks)
out = nets.discriminator(x_fake, y_trg)
loss_fake = adv_loss(out, 0)
loss = porch.sum(loss_real + loss_fake + args.lambda_reg * loss_reg)
return loss, Munch(real=loss_real.numpy().flatten()[0],
fake=loss_fake.numpy().flatten()[0],
reg=loss_reg)
def __iter__(self):
degrees = torch.cat([g.in_degrees().double() ** 0.75 for g in self.graphs])
prob = degrees / torch.sum(degrees)
samples = np.random.choice(
self.length, size=self.num_samples, replace=True, p=prob.numpy()
)
for idx in samples:
yield self.__getitem__(idx)
def forward(self, inputs):
x = self.model(inputs)
gap = porch.nn.functional.adaptive_avg_pool2d(x, 1)
gap_logit = self.gap_fc(gap.view(x.shape[0], -1))
gap_weight = list(self.gap_fc.parameters())[0]
gap = x * porch.Tensor(gap_weight).unsqueeze(0).unsqueeze(3)
gmp = porch.nn.functional.adaptive_max_pool2d(x, 1)
gmp_logit = self.gmp_fc(gmp.view(x.shape[0], -1))
gmp_weight = list(self.gmp_fc.parameters())[0]
gmp = x * porch.Tensor(gmp_weight).unsqueeze(0).unsqueeze(3)
cam_logit = porch.cat([gap_logit, gmp_logit], 1)
x = porch.cat([gap, gmp], 1)
x = self.leaky_relu(self.conv1x1(x))
heatmap = porch.sum(x, dim=1, keepdim=True)
x = self.pad(x)
out = self.conv(x)
return out, cam_logit,heatmap
def sum(self, dim=None, keepdim=False):
return torch.sum(self, dim=dim, keepdim=keepdim)
def normalize(x, eps=1e-10):
return x * torch.rsqrt(torch.sum(x**2, dim=1, keepdim=True) + eps)
def preprocess(x):
"""Preprocess 98-dimensional heatmaps."""
N, C, H, W = x.shape
x = truncate(x)
x = normalize(x)
sw = H // 256
operations = Munch(chin=OPPAIR(0, 3),
eyebrows=OPPAIR(-7 * sw, 2),
nostrils=OPPAIR(8 * sw, 4),
lipupper=OPPAIR(-8 * sw, 4),
liplower=OPPAIR(8 * sw, 4),
lipinner=OPPAIR(-2 * sw, 3))
for part, ops in operations.items():
start, end = index_map[part]
torch.copy(resize(shift(x[:, start:end], ops.shift), ops.resize),
x[:, start:end])
# =
zero_out = torch.cat([
torch.arange(0, index_map.chin.start),
torch.arange(index_map.chin.end, 33),
torch.LongTensor([
index_map.eyebrowsedges.start, index_map.eyebrowsedges.end,
index_map.lipedges.start, index_map.lipedges.end
])
])
x.fill_(val=0, dim=1, indices=zero_out.numpy())
# x_numpy=x.numpy()
# x_numpy[:, zero_out.numpy()] = 0
# x.set_value(x_numpy)
# torch.copy(target)
# x[:, zero_out] = 0
start, end = index_map.nose
torch.copy(shift(x[:, start + 1:end], 4 * sw), x[:, start + 1:end])
torch.copy(resize(x[:, start:end], 1), x[:, start:end])
# x[:, start+1:end] = shift(x[:, start+1:end], 4*sw)
# x[:, start:end] = resize(x[:, start:end], 1)
start, end = index_map.eyes
torch.copy(resize(x[:, start:end], 1), x[:, start:end])
torch.copy(resize(shift(x[:, start:end], -8), 3) + \
shift(x[:, start:end], -24), x[:, start:end] )
# x[:, start:end] = resize(x[:, start:end], 1)
# x[:, start:end] = resize(shift(x[:, start:end], -8), 3) + \
# shift(x[:, start:end], -24)
# Second-level mask
x2 = x.clone() #deepcopy(x)
x2.fill_(0, 1, list(range(index_map.chin.start, index_map.chin.end)))
x2.fill_(0, 1, list(range(index_map.lipedges.start,
index_map.lipedges.end)))
x2.fill_(0, 1, list(range(index_map.eyebrows.start,
index_map.eyebrows.end)))
# x2[:, index_map.chin.start:index_map.chin.end] = 0 # start:end was 0:33
# x2[:, index_map.lipedges.start:index_map.lipinner.end] = 0 # start:end was 76:96
# x2[:, index_map.eyebrows.start:index_map.eyebrows.end] = 0 # start:end was 33:51
x = torch.sum(x, dim=1, keepdim=True) # (N, 1, H, W)
x2 = torch.sum(x2, dim=1, keepdim=True) # mask without faceline and mouth
x_numpy = x.numpy
zero_indices = np.where(x_numpy != x_numpy)[0]
x.fill_(0, 0, zero_indices)
x2.fill_(0, 0, zero_indices)
# x[x != x] = 0 # set nan to zero
# x2[x != x] = 0 # set nan to zero
return x.clamp_(0, 1), x2.clamp_(0, 1)
