def heatmaps_to_coords(xy_hm, zy_hm, xz_hm):
xy = dsnt(xy_hm)
zy = dsnt(zy_hm)
xz = dsnt(xz_hm)
x, y = xy.split(1, -1)
z = 0.5 * (zy[:, :, 0:1] + xz[:, :, 1:2])
return torch.cat([x, y, z], -1)
def forward(self, sag_x, cor_x):
#* Output are features at every spatial resolution
sag_out = self.encoder.forward(sag_x)
cor_out = self.encoder.forward(cor_x)
if self.classifier is not None:
sag_class = self.classifier(sag_out[-1])
cor_class = self.classifier(cor_out[-1])
# print(sag_class.shape, cor_class.shape)
mean_class = (sag_class + cor_class) / 2
#* Combine Sagittal + coronal at each resolution
output = [
torch.cat([sag, cor], dim=1) for sag, cor in zip(sag_out, cor_out)
]
#output = [sag*cor for sag, cor in zip(sag_out, cor_out)]
#* Upscale to match input spatial resolution
decoder_output = self.decoder.forward(*output)
#* Get correct number of channels
seg_out = self.segmentation_head.forward(decoder_output)
#* Convert to 1D heatmap
unnorm_heatmap = self.hm_conv(seg_out)
heatmap = dsntnn.flat_softmax(unnorm_heatmap[..., 0])
coords = dsntnn.dsnt(heatmap, normalized_coordinates=True)
return seg_out, heatmap, coords[..., 0], mean_class
def forward(self, x):
unnormalized_heatmaps = self.heatmap_conv(x)
# Softmax converts values in (0,1) range, so no ReLu needed to remove negative values.
normalized_heatmaps = dsntnn.flat_softmax(unnormalized_heatmaps) # make it a prob distribution
coords = dsntnn.dsnt(normalized_heatmaps)
return coords, normalized_heatmaps, unnormalized_heatmaps
def forward(self, *inputs):
t = inputs[0]
t = self.in_cnn(t)
self.xy_heatmaps = [flat_softmax(self.xy_hm_cnn(t))]
self.zy_heatmaps = [flat_softmax(self.zy_hm_cnn(t))]
self.xz_heatmaps = [flat_softmax(self.xz_hm_cnn(t))]
xy = dsnt(self.xy_heatmaps[-1])
zy = dsnt(self.zy_heatmaps[-1])
xz = dsnt(self.xz_heatmaps[-1])
x = xy.narrow(-1, 0, 1)
y = xy.narrow(-1, 1, 1)
z = 0.5 * (zy.narrow(-1, 0, 1) + xz.narrow(-1, 1, 1))
return torch.cat([x, y, z], -1)
def test_dsnt_backward():
in_var = SIMPLE_INPUT.detach().requires_grad_(True)
output = dsnt(in_var)
loss = mse_loss(output, SIMPLE_TARGET)
loss.backward()
assert_allclose(in_var.grad, SIMPLE_GRAD_INPUT)
def forward(self, x, b_prev, h_prev, c_prev, centers):
f = self.encode(x)
f = torch.cat([f, b_prev, centers], dim=1)
h, c = self.lstm(f, h_prev, c_prev)
b = self.generate(h)
b = dsntnn.flat_softmax(b)
coords = dsntnn.dsnt(b)
return b, h, c, coords
def test_backward(self):
mse = torch.nn.MSELoss()
in_var = self.SIMPLE_INPUT.detach().requires_grad_(True)
output = dsnt(in_var)
loss = mse(output, self.SIMPLE_TARGET)
loss.backward()
self.assertEqual(in_var.grad, self.SIMPLE_GRAD_INPUT)
def test_sharpened_dsnt_forward():
inp = torch.tensor([[[
[0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.1, 0.0],
[0.1, 0.0, 0.0, 0.6, 0.1],
[0.0, 0.0, 0.0, 0.1, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0],
]]])
expected = torch.tensor([[[0.399983, 0.0]]])
actual = dsnt(sharpen_heatmaps(inp, alpha=6))
assert_allclose(actual, expected)
def forward(self, images):
# 1. Run the images through our Resnet
resnet_out = self.resnet(images)
# 2. Use a 1x1 conv to get one unnormalized heatmap per location
unnormalized_heatmaps = self.hm_conv(resnet_out)
# 3. Normalize the heatmaps
heatmaps = dsntnn.flat_softmax(unnormalized_heatmaps)
# 4. Calculate the coordinates
coords = dsntnn.dsnt(heatmaps)
return coords, heatmaps
def forward(self, images):
# run images trough FCN
fcn_out = self.fcn(images)
# use a 1x1 conv to get one unnormalized heatmap per location
unnormalized_heatmaps = self.hm_conv(fcn_out)
# normalize the heatmaps
heatmaps = dsntnn.flat_softmax(unnormalized_heatmaps)
# calculate the coordinates
coords = dsntnn.dsnt(heatmaps)
return coords, heatmaps
def forward(self, images):
# Run a frame through our network
out = self.resnet(images)
# Use a 1x1 conv to get one unnormalized heatmap
# per location
unnormalized_heatmaps = self.hm_conv(out)
# Normalize the heatmaps
heatmaps = dsntnn.flat_softmax(unnormalized_heatmaps)
# Perform coordinate regression
coords = dsntnn.dsnt(heatmaps)
return coords, heatmaps
def test_backward(self):
mse = torch.nn.MSELoss()
in_var = Variable(self.SIMPLE_INPUT, requires_grad=True)
output = dsnt(in_var)
target_var = Variable(self.SIMPLE_TARGET, requires_grad=False)
loss = mse(output, target_var)
loss.backward()
expected = self.SIMPLE_GRAD_INPUT
actual = in_var.grad.data
self.assertEqual(actual, expected)
def test_cuda(self):
mse = torch.nn.MSELoss()
in_var = Variable(self.SIMPLE_INPUT.cuda(), requires_grad=True)
expected_output = self.SIMPLE_OUTPUT.cuda()
output = dsnt(in_var)
self.assertEqual(output.data, expected_output)
target_var = Variable(self.SIMPLE_TARGET.cuda(), requires_grad=False)
loss = mse(output, target_var)
loss.backward()
expected_grad = self.SIMPLE_GRAD_INPUT.cuda()
self.assertEqual(in_var.grad.data, expected_grad)
def forward(self, h):
# 1. Use a 1x1 conv to get one unnormalized heatmap per location
unnormalized_heatmaps = self.hm_conv(h)
# 2. Transpose the heatmap volume to keep the temporal dimension in the volume
unnormalized_heatmaps.transpose_(2, 1).transpose_(1, 0)
# 3. Normalize the heatmaps
heatmaps = [dsntnn.flat_softmax(uhm) for uhm in unnormalized_heatmaps]
# heatmaps = dsntnn.flat_softmax(unnormalized_heatmaps)
# 4. Calculate the coordinates
# coords = dsntnn.dsnt(heatmaps)
coords = [dsntnn.dsnt(hm) for hm in heatmaps]
heatmaps = torch.stack(heatmaps, 1)
coords = torch.stack(coords, 1)
return coords, heatmaps
def test_cuda(self):
mse = torch.nn.MSELoss()
in_var = self.SIMPLE_INPUT.detach().cuda().requires_grad_(True)
expected_output = self.SIMPLE_OUTPUT.cuda()
output = dsnt(in_var)
self.assertEqual(output, expected_output)
target_var = self.SIMPLE_TARGET.cuda()
loss = mse(output, target_var)
loss.backward()
expected_grad = self.SIMPLE_GRAD_INPUT.cuda()
self.assertEqual(in_var.grad, expected_grad)
def test_dsnt_cuda():
mse = torch.nn.MSELoss()
in_var = SIMPLE_INPUT.detach().cuda().requires_grad_(True)
expected_output = SIMPLE_OUTPUT.cuda()
output = dsnt(in_var)
assert_allclose(output, expected_output)
target_var = SIMPLE_TARGET.cuda()
loss = mse(output, target_var)
loss.backward()
expected_grad = SIMPLE_GRAD_INPUT.cuda()
assert_allclose(in_var.grad, expected_grad)
def test_3d(self):
inp = torch.Tensor([[[[
[0.00, 0.00, 0.00],
[0.00, 0.00, 0.00],
[0.00, 0.00, 0.00],
], [
[0.00, 0.00, 0.00],
[0.00, 0.00, 0.00],
[1.00, 0.00, 0.00],
], [
[0.00, 0.00, 0.00],
[0.00, 0.00, 0.00],
[0.00, 0.00, 0.00],
]]]])
expected = torch.Tensor([[[-2 / 3, 2 / 3, 0]]])
self.assertEqual(dsnt(inp), expected)
def test_dsnt_3d():
inp = torch.tensor([[[[
[0.00, 0.00, 0.00],
[0.00, 0.00, 0.00],
[0.00, 0.00, 0.00],
], [
[0.00, 0.00, 0.00],
[0.00, 0.00, 0.00],
[1.00, 0.00, 0.00],
], [
[0.00, 0.00, 0.00],
[0.00, 0.00, 0.00],
[0.00, 0.00, 0.00],
]]]])
expected = torch.tensor([[[-2 / 3, 2 / 3, 0]]])
assert_allclose(dsnt(inp), expected)
def forward(self, images):
images = images.cuda()
# 1. Run the images through our FCN
fcn_out = self.fcn(images)
# 2. Use a 1x1 conv to get one unnormalized heatmap per location
unnormalized_heatmaps = self.hm_conv(fcn_out)
# 3. Normalize the heatmaps
heatmaps = dsntnn.flat_softmax(unnormalized_heatmaps)
# 4. Calculate the coordinates
coords = dsntnn.dsnt(heatmaps)
return coords, heatmaps
#
# import traf_data
# import os
# import cv2
# import numpy as np
# import torch
# from tensorboardX import SummaryWriter
#
# image_size = [800, 400]
#
# train_path = "/home/asprohy/data/traffic"
# train_data,_,_ = traf_data.get_data2(train_path)
#
# img = cv2.imread(os.path.join(train_path, train_data[0]['filepath']))
# print(os.path.join(train_path, train_data[0]['filepath']))
# h, w = img.shape[:2]
# print('h[],w[]', h, w)
# print('filepath',train_data[0]['filepath'])
# # print(img)
# img = cv2.resize(img, (image_size[0],image_size[1]))
# img = np.array(img)
# tmpc = train_data[0]['bboxes'][0]
# print('lab', tmpc)
# label_all = [int((tmpc['x1'] + tmpc['x2'])/2 / w * image_size[0]),int((tmpc['y1'] + tmpc['y2'])/2 / h * image_size[1])]
# print('lab', label_all)
# raccoon_face_tensor = torch.from_numpy(img).permute(2, 0, 1).float()
# input_tensor = raccoon_face_tensor.div(255).unsqueeze(0)
# input_var = input_tensor
#
# model = CoordRegressionNetwork(n_locations=1)
#
# with SummaryWriter(comment='Net1')as w:
#
# w.add_graph(model, (input_var,))
def forward(self, sag_x, cor_x):
sag_out = self.encoder(sag_x)
cor_out = self.encoder(cor_x)
print(len(sag_out))
print(sag_out[0].shape)
seg_out = self.head(self.decoder(torch.mul(sag_out['out'], cor_out['out'])))
print(seg_out.shape)
# 2. Use a 1x1 conv to get one unnormalized heatmap per location
unnorm_heatmap = self.hm_conv(seg_out) # 1-Dim Heatmap (BxCxHx1)
print(unnorm_heatmap.shape)
# 3. Softmax across HxW
heatmap = dsntnn.flat_softmax(unnorm_heatmap)
# Coordinate from heatmap (Return )
coords= dsntnn.dsnt(heatmap, normalized_coordinates=False)
#Global Average pooling
pred = F.avg_pool2d(seg_out, kernel_size=seg_out.size()[2:])
return unnorm_heatmap, seg_out, coords[..., 1], pred
def forward_part2(self, x):
"""Forward from unnormalized heatmaps to output"""
if self.output_strat == 'dsnt':
x = self._hm_preact(x, self.preact)
self.heatmaps = x
x = dsnt(x)
# elif self.output_strat == 'fc':
# x = self._hm_preact(x, self.preact)
# self.heatmaps = x
# height = x.size(-2)
# width = x.size(-1)
# x = x.view(-1, height * width)
# x = self.out_fc(x)
# x = x.view(-1, self.n_chans, 2)
# else:
# self.heatmaps = x
return x
def forward(self, h):
probabilities = torch.zeros(
0, device=h.device
) # torch.nn.ReLU(self.probability(torch.squeeze(h)))
# 1. Use a 1x1 conv to get one unnormalized heatmap per location
if self.temporal_interpolate > 1:
h = F.interpolate(h,
scale_factor=(self.temporal_interpolate, 1, 1),
mode='trilinear')
unnormalized_heatmaps = self.hm_conv(h)
# 2. Transpose the heatmap volume to keep the temporal dimension in the volume
unnormalized_heatmaps.transpose_(2, 1).transpose_(1, 0)
# 3. Normalize the heatmaps
heatmaps = [dsntnn.flat_softmax(uhm) for uhm in unnormalized_heatmaps]
# 4. Calculate the coordinates
coords = [dsntnn.dsnt(hm) for hm in heatmaps]
heatmaps = torch.stack(heatmaps, 1)
coords = torch.stack(coords, 1)
return coords, heatmaps, probabilities
def forward(self, images, sample=False):
if sample:
xs = self.fcn(images, sample=sample)
return xs
else:
fcn_out, dec_out, xhist = self.fcn(images)
unnormalized_heatmaps = self.hm_conv(fcn_out)
clean_output = self.hm_conv2(dec_out)
heatmaps = dsntnn.flat_softmax(unnormalized_heatmaps)
coords = dsntnn.dsnt(heatmaps)
# input_edges=self.relu(nn.functional.conv2d(images,self.f))
# input_edges=torch.add(torch.add(input_edges[:,0:1,:,:],input_edges[:,1:2,:,:]),torch.add(input_edges[:,2:3,:,:],input_edges[:,3:4,:,:]))
# clean_edges=self.relu(nn.functional.conv2d(clean_output,self.f))
# clean_edges=torch.add(torch.add(clean_edges[:,0:1,:,:],clean_edges[:,1:2,:,:]),torch.add(clean_edges[:,2:3,:,:],clean_edges[:,3:4,:,:]))
# mult=torch.mul(clean_edges,input_edges)
# out_noise=(torch.cat([self.noise(mult),nn.functional.conv2d(mult,self.f2)],dim=1))
# out_ridge=(torch.cat([self.ridge(mult),nn.functional.conv2d(mult,self.f1)],dim=1))
# ridge=self.maxpool1(out_ridge)
# noise=self.maxpool2(out_noise)
return coords, heatmaps, clean_output, xhist
def forward(self, x):
b = self.process(x)
b = dsntnn.flat_softmax(b)
c = dsntnn.dsnt(b)
return b, c
def op(inp):
return dsnt(inp, normalized_coordinates=False)
def test_dsnt_forward_not_normalized():
expected = torch.tensor([[[3.0, 2.0]]])
actual = dsnt(SIMPLE_INPUT, normalized_coordinates=False)
assert_allclose(actual, expected)
def test_dsnt_forward():
expected = SIMPLE_OUTPUT
actual = dsnt(SIMPLE_INPUT)
assert_allclose(actual, expected)
def forward(self, x):
x = flat_softmax(x)
x = dsnt(x, normalized_coordinates=True)
return x
def forward(self, images):
fcn_out, xhist = self.fcn(images)
unnormalized_heatmaps = self.hm_conv(fcn_out)
heatmaps = dsntnn.flat_softmax(unnormalized_heatmaps)
coords = dsntnn.dsnt(heatmaps)
return coords, heatmaps, xhist
def test_forward(self):
expected = self.SIMPLE_OUTPUT
actual = dsnt(self.SIMPLE_INPUT)
self.assertEqual(actual, expected)
