def convlstm_forward(x, CLSTM):
len_r = len(reset_group)
if len_r > 0:
yyy = []
if reset_group[0][0] > 0:
# print(reset_group[0][0])
temp_x = x[0:reset_group[0][0]]
xx = temp_x.view(-1, temp_x.size(0), temp_x.size(1),
temp_x.size(2), temp_x.size(3))
xxx = CLSTM(xx)
yyy.append(xxx[1])
for item in reset_group:
# print (item)
temp_x = x[item[0]:item[1]]
xx = temp_x.view(-1, temp_x.size(0), temp_x.size(1),
temp_x.size(2), temp_x.size(3))
xxx = CLSTM(xx)
yyy.append(xxx[1])
yyy = torch.cat(yyy, 0)
yyy = F.relu(yyy, inplace=True)
yyy = yyy.view(x.size())
return yyy
else:
xx = x.view(-1, x.size(0), x.size(1), x.size(2), x.size(3))
xxx = CLSTM(xx)
yyy = xxx[1]
yyy = F.relu(yyy, inplace=True)
yyy = yyy.view(x.size())
return yyy
def __init__(self, base, extras, extras_refinement, head, head_r,
num_classes): #, use_gru = False
super(RefineSSD, self).__init__()
self.num_classes = num_classes
self.binary_classes = 2
# TODO: implement __call__ in PriorBox
self.priorbox = PriorBox(v2)
self.priors = Variable(self.priorbox.forward(), volatile=True)
self.num_priors = self.priors.size(0)
self.size = 300
self.hidden_states = [[[], [], [], [], [], []], [[], [], [], [], [],
[]],
[[], [], [], [], [], []], [[], [], [], [], [],
[]]]
# SSD network
self.vgg = nn.ModuleList(base)
self.clstm_1 = nn.ModuleList([CLSTM(512, 512, 3, stride=1, padding=1)])
self.clstm_2 = nn.ModuleList(
[CLSTM(1024, 1024, 3, stride=1, padding=1)])
# Layer learns to scale the l2 normalized features from conv4_3
self.L2Norm = L2Norm(512, 20)
self.L2Norm_r = L2Norm(512, 20)
self.extras = nn.ModuleList(extras)
self.extras_r = nn.ModuleList(extras_refinement)
self.loc = nn.ModuleList(head[0])
self.conf = nn.ModuleList(head[1])
self.loc_r = nn.ModuleList(head_r[0])
self.conf_r = nn.ModuleList(head_r[1])
self.softmax = nn.Softmax().cuda()
def convlstm_forward(x, CLSTM, hidden_states, hi):
len_r = len(reset_group)
if len_r > 0:
yyy = []
if reset_group[0][0] > 0:
temp_x = x[0:reset_group[0][0]]
xx = temp_x.view(-1, temp_x.size(0), temp_x.size(1),
temp_x.size(2), temp_x.size(3))
len_hid = len(hidden_states[hi])
if len_hid > 0:
temp = [(Variable(hidden_states[hi][0]).cuda(),
Variable(hidden_states[hi][1]).cuda())]
xxx = CLSTM(xx, temp)
else:
xxx = CLSTM(xx)
hidden_states[hi] = [xxx[0][0][0].data, xxx[0][0][1].data]
yyy.append(xxx[1].data)
for item in reset_group:
hidden_states[hi] = []
temp_x = x[item[0]:item[1]]
xx = temp_x.view(-1, temp_x.size(0), temp_x.size(1),
temp_x.size(2), temp_x.size(3))
xxx = CLSTM(xx)
hidden_states[hi] = [xxx[0][0][0].data, xxx[0][0][1].data]
yyy.append(xxx[1].data)
third_tensor = yyy[0]
if len(yyy) > 1:
for yitem in yyy[1:]:
third_tensor = torch.cat((third_tensor, yitem), 0)
yyyy = Variable(third_tensor).cuda()
yyyy = F.relu(yyyy, inplace=True)
x = yyyy.view(x.size())
return x
else:
xx = x.view(-1, x.size(0), x.size(1), x.size(2), x.size(3))
len_hid = len(hidden_states[hi])
if len_hid > 0:
temp = [(Variable(hidden_states[hi][0]).cuda(),
Variable(hidden_states[hi][1]).cuda())]
xxx = CLSTM(xx, temp)
else:
xxx = CLSTM(xx)
yyy = xxx[1]
hidden_states[hi] = [xxx[0][0][0].data, xxx[0][0][1].data]
yyyy = F.relu(yyy, inplace=True)
x = yyyy.view(x.size())
return x
def add_extras_recurrent():
# recurrent Extra layers added to VGG for feature scaling
layers = []
# [256, 'S', 512, 128, 'S', 256, 128, 256, 128, 256]
# layers += [nn.Conv2d(1024, 256, kernel_size=(1, 1))]
layers += [CLSTM(256, 512, filter_size=3, stride=2, padding=1)]
# layers += [nn.Conv2d(512, 128, kernel_size=(1, 1))]
layers += [CLSTM(128, 256, filter_size=3, stride=2, padding=1)]
# layers += [nn.Conv2d(256, 128, kernel_size=(1, 1))]
layers += [CLSTM(128, 256, filter_size=3, stride=1)]
# layers += [nn.Conv2d(256, 128, kernel_size=(1, 1))]
layers += [CLSTM(128, 256, filter_size=3, stride=1)]
return layers
def __init__(self, base, extras, head, num_classes, use_gru = False):
super(SSD, self).__init__()
self.num_classes = num_classes
# TODO: implement __call__ in PriorBox
self.priorbox = PriorBox(v2)
self.priors = Variable(self.priorbox.forward(), volatile=True)
self.num_priors = self.priors.size(0)
self.size = 300
self.hidden_states = [[]]
# SSD network
self.vgg = nn.ModuleList(base)
clstm = [CLSTM(512, 512, 3, 1)]
cgru = [CGRU(512, 512, 3, 1)]
self.clstm = nn.ModuleList(clstm)
self.use_gru = use_gru
if self.use_gru:
self.cgru = nn.ModuleList(cgru)
print ("ConvGRU is used")
else:
print("ConvLSTM is used")
# if self.use_gru:
# self.hidden_states = []
# Layer learns to scale the l2 normalized features from conv4_3
self.L2Norm = L2Norm(512, 20)
self.extras = nn.ModuleList(extras)
self.loc = nn.ModuleList(head[0])
self.conf = nn.ModuleList(head[1])
self.softmax = nn.Softmax().cuda()
def multibox(vgg, extra_layers, cfg, num_classes):
loc_layers = []
conf_layers = []
vgg_source = [24, -2]
for k, v in enumerate(vgg_source):
loc_layers += [
nn.Conv2d(vgg[v].out_channels,
cfg[k] * 4,
kernel_size=3,
padding=1)
]
conf_layers += [
nn.Conv2d(vgg[v].out_channels,
cfg[k] * num_classes,
kernel_size=3,
padding=1)
]
for k, v in enumerate(extra_layers[1::2], 2):
loc_layers += [
nn.Conv2d(v.out_channels, cfg[k] * 4, kernel_size=3, padding=1)
]
conf_layers += [
nn.Conv2d(v.out_channels,
cfg[k] * num_classes,
kernel_size=3,
padding=1)
]
clstm = [CLSTM(512, 512, 3, 1)]
cgru = [CGRU(512, 512, 3, 1)]
return vgg, extra_layers, clstm, cgru, (loc_layers, conf_layers)
def __init__(self, base, extras, head, num_classes):
super(SSD, self).__init__()
self.num_classes = num_classes
# TODO: implement __call__ in PriorBox
self.priorbox = PriorBox(v2)
self.priors = Variable(self.priorbox.forward(), volatile=True).cuda()
self.num_priors = self.priors.size(0)
self.size = 300
# SSD network
self.vgg = nn.ModuleList(base)
# Layer learns to scale the l2 normalized features from conv4_3
clstm = [CLSTM(512, 512, 3, 1)]
self.clstm = nn.ModuleList(clstm)
self.L2Norm = L2Norm(512, 20)
self.extras = nn.ModuleList(extras)
self.loc = nn.ModuleList(head[0])
self.conf = nn.ModuleList(head[1])
self.softmax = nn.Softmax().cuda()