def __init__(self,
n_classes,
deno,
in_out = None,
# feat_dim = None,
sparsify = 0.5,
aft_nonlin = 'RL',
sigmoid = False,
feat_ret= False):
super(Graph_Multi_Video, self).__init__()
self.num_classes = n_classes
self.feat_ret = feat_ret
self.deno = deno
self.graph_size = 1
self.sparsify = sparsify
if in_out is None:
in_out = [2048,512]
# if feat_dim is None:
# feat_dim = [2048,256]
# self.linear_layer = nn.Linear(feat_dim[0], feat_dim[1], bias = True)
self.graph_layer = Graph_Layer_Wrapper(in_out[0],n_out = in_out[1], aft_nonlin = aft_nonlin, type_layer = 'cooc')
last_graph = []
last_graph.append(nn.Dropout(0.5))
last_graph.append(nn.Linear(in_out[-1],n_classes, bias = True))
self.last_graph = nn.Sequential(*last_graph)
def __init__(self,
n_classes,
deno,
in_out = None,
# feat_dim = None,
sparsify = 0.5,
aft_nonlin = 'RL',
sigmoid = False,
feat_ret= False):
super(Graph_Multi_Video, self).__init__()
self.num_classes = n_classes
self.feat_ret = feat_ret
self.deno = deno
self.graph_size = 1
self.sparsify = sparsify
if in_out is None:
in_out = [2048,64]
# if feat_dim is None:
# feat_dim = [2048,256]
# self.linear_layer = nn.Linear(feat_dim[0], feat_dim[1], bias = True)
self.graph_layers = nn.ModuleList()
for class_num in range(self.num_classes):
self.graph_layers.append(Graph_Layer_Wrapper(in_out[0],n_out = in_out[1], aft_nonlin = aft_nonlin, type_layer = 'cooc', non_lin = None))
# self.graph_layers = nn.ModuleList(*self.graph_layers)
# self.last_graph = nn.ModuleList()
# for class_num in range(self.num_classes):
last_graph = []
# if aft_nonlin is not None:
# to_pend = aft_nonlin.split('_')
# for tp in to_pend:
# if tp.lower()=='ht':
# last_graph.append(nn.Hardtanh())
# elif tp.lower()=='rl':
# last_graph.append(nn.ReLU())
# elif tp.lower()=='l2':
# last_graph.append(Normalize())
# elif tp.lower()=='ln':
# last_graph.append(nn.LayerNorm(n_out))
# elif tp.lower()=='bn':
# last_graph.append(nn.BatchNorm1d(n_out, affine = False, track_running_stats = False))
# elif tp.lower()=='sig':
# last_graph.append(nn.Sigmoid())
# else:
# error_message = str('non_lin %s not recognized', non_lin)
# raise ValueError(error_message)
last_graph.append(nn.Dropout(0.5))
last_graph.append(nn.Linear(in_out[-1]*self.num_classes,self.num_classes, bias = True))
if sigmoid:
last_graph.append(nn.Sigmoid())
self.last_graph = nn.Sequential(*last_graph)
def __init__(self,
n_classes,
deno,
pretrained,
in_out=None,
graph_size=None,
method='cos'):
super(Graph_Multi_Video, self).__init__()
self.num_classes = n_classes
self.deno = deno
self.graph_size = graph_size
if in_out is None:
in_out = [2048, 64, 2048, 64]
num_layers = len(in_out) - 3
non_lin = 'HT'
print 'NUM LAYERS', num_layers, in_out
self.linear_layer = nn.Linear(in_out[0], in_out[1], bias=False)
# for param in self.linear_layer.parameters():
# param.requires_grad = False
non_lin = 'HT'
if pretrained == 'ucf':
model_file = '../experiments/just_mill_flexible_deno_8_n_classes_20_layer_sizes_2048_64_ucf/all_classes_False_just_primary_False_limit_500_cw_True_MultiCrossEntropy_100_step_100_0.1_0.001_0.001/model_99.pt'
elif pretrained == 'activitynet':
model_file = '../experiments/just_mill_flexible_deno_8_n_classes_100_layer_sizes_2048_64_activitynet/all_classes_False_just_primary_False_limit_500_cw_True_MultiCrossEntropy_50_step_50_0.1_0.001_0.001/model_49.pt'
elif pretrained == 'random':
model_file = '../experiments/just_mill_flexible_deno_8_n_classes_20_layer_sizes_2048_64_ucf/all_classes_False_just_primary_False_limit_500_cw_True_MultiCrossEntropy_100_step_100_0.1_0_0.001/model_99.pt'
else:
error_message = 'Similarity method %s not valid' % method
raise ValueError(error_message)
model_temp = torch.load(model_file)
self.linear_layer.weight.data = model_temp.linear.weight.data
self.graph_layers = nn.ModuleList()
for num_layer in range(num_layers):
self.graph_layers.append(
Graph_Layer_Wrapper(in_out[num_layer + 2],
n_out=in_out[num_layer + 3],
non_lin=non_lin,
method=method))
last_layer = []
last_layer.append(nn.Hardtanh())
last_layer.append(Normalize())
last_layer.append(nn.Dropout(0.5))
last_layer.append(nn.Linear(in_out[-1], n_classes))
last_layer = nn.Sequential(*last_layer)
self.last_layer = last_layer
def __init__(self,
n_classes,
deno,
in_out=None,
feat_dim=None,
graph_size=None,
method='cos',
sparsify=0.5,
non_lin='RL',
aft_nonlin='RL',
sigmoid=False,
layer_bef=None,
graph_sum=False,
background=False,
just_graph=False,
feat_ret=False,
dropout=0.5):
super(Graph_Multi_Video, self).__init__()
self.num_classes = n_classes
self.background = background
self.feat_ret = feat_ret
if self.background:
assert sigmoid
n_classes += 1
self.deno = deno
self.graph_size = graph_size
self.sparsify = sparsify
self.graph_sum = graph_sum
self.just_graph = just_graph
if in_out is None:
in_out = [2048, 512]
if feat_dim is None:
feat_dim = [2048, 256]
num_layers = 1
print 'NUM LAYERS', num_layers, in_out
self.bn = None
# nn.BatchNorm1d(2048, affine = False)
self.linear_layer = []
self.linear_layer.append(nn.Linear(feat_dim[0], feat_dim[1],
bias=True))
if non_lin is not None:
if non_lin.lower() == 'ht':
self.linear_layer.append(nn.Hardtanh())
elif non_lin.lower() == 'rl':
self.linear_layer.append(nn.ReLU())
else:
error_message = str('non_lin %s not recognized', non_lin)
raise ValueError(error_message)
# self.linear_layer = nn.Sequential(*self.linear_layer)
# self.graph_layer = nn.ModuleList()
# self.last_graph = nn.ModuleList()
self.graph_layer = Graph_Layer_Wrapper(in_out[0],
n_out=in_out[1],
non_lin=None,
method=method,
aft_nonlin=aft_nonlin)
last_graph = []
last_graph.append(nn.Dropout(dropout))
last_graph.append(nn.Linear(in_out[-1], n_classes, bias=True))
if sigmoid:
last_graph.append(nn.Sigmoid())
self.last_graph = nn.Sequential(*last_graph)
class Graph_Multi_Video(nn.Module):
def __init__(self,
n_classes,
deno,
in_out=None,
feat_dim=None,
graph_size=None,
method='cos',
sparsify=0.5,
non_lin='RL',
aft_nonlin='RL',
sigmoid=False,
layer_bef=None,
graph_sum=False,
background=False,
just_graph=False,
feat_ret=False,
dropout=0.5):
super(Graph_Multi_Video, self).__init__()
self.num_classes = n_classes
self.background = background
self.feat_ret = feat_ret
if self.background:
assert sigmoid
n_classes += 1
self.deno = deno
self.graph_size = graph_size
self.sparsify = sparsify
self.graph_sum = graph_sum
self.just_graph = just_graph
if in_out is None:
in_out = [2048, 512]
if feat_dim is None:
feat_dim = [2048, 256]
num_layers = 1
print 'NUM LAYERS', num_layers, in_out
self.bn = None
# nn.BatchNorm1d(2048, affine = False)
self.linear_layer = []
self.linear_layer.append(nn.Linear(feat_dim[0], feat_dim[1],
bias=True))
if non_lin is not None:
if non_lin.lower() == 'ht':
self.linear_layer.append(nn.Hardtanh())
elif non_lin.lower() == 'rl':
self.linear_layer.append(nn.ReLU())
else:
error_message = str('non_lin %s not recognized', non_lin)
raise ValueError(error_message)
# self.linear_layer = nn.Sequential(*self.linear_layer)
# self.graph_layer = nn.ModuleList()
# self.last_graph = nn.ModuleList()
self.graph_layer = Graph_Layer_Wrapper(in_out[0],
n_out=in_out[1],
non_lin=None,
method=method,
aft_nonlin=aft_nonlin)
last_graph = []
last_graph.append(nn.Dropout(dropout))
last_graph.append(nn.Linear(in_out[-1], n_classes, bias=True))
if sigmoid:
last_graph.append(nn.Sigmoid())
self.last_graph = nn.Sequential(*last_graph)
def forward(self, input, epoch_num=None, ret_bg=False, branch_to_test=-1):
strip = False
if type(input) != type([]):
input = [input]
strip = True
identity = False
method = None
# print 'self.graph_size' , self.graph_size
if not self.training:
graph_size = 1
else:
if self.graph_size is None:
graph_size = len(input)
elif self.graph_size == 'rand':
graph_size = random.randint(1, len(input))
elif type(self.graph_size) == str and self.graph_size.startswith(
'randupto'):
graph_size = int(self.graph_size.split('_')[1])
graph_size = random.randint(1, min(graph_size, len(input)))
else:
graph_size = min(self.graph_size, len(input))
# print 'graph_size', graph_size, self.training
input_chunks = [[input[0]]] + [
input[i:i + graph_size] for i in xrange(1, len(input), graph_size)
]
# input_chunks = [input[i:i + graph_size] for i in xrange(0, len(input), graph_size)]
is_cuda = next(self.parameters()).is_cuda
# print 'Graph branch'
# pmf_all = [[] for i in range(self.num_branches)]
# x_all_all = [[] for i in range(self.num_branches)]
pmf_all = []
x_all = []
graph_sums = []
out_graph_all = []
input_sizes_all = np.array(
[input_curr.size(0) for input_curr in input])
for input in input_chunks:
input_sizes = [input_curr.size(0) for input_curr in input]
# print input_sizes
input = torch.cat(input, 0)
# print input.size()
if is_cuda:
input = input.cuda()
# assert len(self.graph_layers)==(self.num_branches)
# if hasattr(self, 'layer_bef') and self.layer_bef is not None:
# input = self.layer_bef(input)
# feature_out = self.linear_layer(input)
# # for col_num in range(len(self.graph_layers)):
# print 'feature_out.size()',feature_out.size()
to_keep = self.sparsify
# if to_keep=='lin':
# out_graph = self.graph_layer(input)
# else:
out_graph = self.graph_layer(input,
input,
to_keep=to_keep,
graph_sum=self.graph_sum,
identity=identity,
method=method)
if self.graph_sum:
[out_graph, graph_sum] = out_graph
graph_sums.append(graph_sum.unsqueeze(0))
# out_graph_all.append(out_graph)
# print 'out_graph.size()',out_graph.size()
out_col = self.last_graph(out_graph)
# print 'out_col.size()',out_col.size()
# x_all.append(out_col)
# x = x_all[-1]
for idx_sample in range(len(input_sizes)):
if idx_sample == 0:
start = 0
else:
start = sum(input_sizes[:idx_sample])
end = start + input_sizes[idx_sample]
x_curr = out_col[start:end, :]
# THIS LINE IS DIFFERENT BETWEEN RETF AND NO RETF
out_graph_curr = feature_out[start:end, :]
x_all.append(x_curr)
out_graph_all.append(out_graph_curr)
pmf_all += [self.make_pmf(x_curr).unsqueeze(0)]
if strip:
# for idx_pmf, pmf in enumerate(pmf_all):
# assert len(pmf)==1
pmf_all = pmf_all[0].squeeze()
x_all = x_all[0]
# print torch.min(pmf_all), torch.max(pmf_all)
# for idx_x, x in enumerate(x_all):
# x_all = torch.cat(x_all,dim=0)
# out_graph_all = torch.cat(out_graph_all, dim = 0)
# print x_all.size(), out_graph_all.size()
# print graph_sums
if hasattr(self, 'feat_ret') and self.feat_ret:
pmf_all = [
pmf_all,
[torch.cat(graph_sums, dim=0), out_graph_all, input_sizes_all]
]
elif self.graph_sum:
pmf_all = [pmf_all, torch.cat(graph_sums, dim=0)]
# raw_input()
if ret_bg:
return x_all, pmf_all, None
else:
return x_all, pmf_all
def make_pmf(self, x):
k = max(1, x.size(0) // self.deno)
pmf, _ = torch.sort(x, dim=0, descending=True)
pmf = pmf[:k, :]
pmf = torch.sum(pmf[:k, :], dim=0) / k
return pmf
def get_similarity(self,
input,
idx_graph_layer=0,
sparsify=False,
nosum=True):
# if sparsify is None:
# sparsify = self.sparsify
is_cuda = next(self.parameters()).is_cuda
# input_sizes = [input_curr.size(0) for input_curr in input]
# input = torch.cat(input,0)
if is_cuda:
input = input.cuda()
# assert idx_graph_layer<len(self.graph_layers)
if hasattr(self, 'layer_bef') and self.layer_bef is not None:
input = self.layer_bef(input)
# feature_out = self.linear_layer(input)
if sparsify:
to_keep = self.sparsify
# [idx_graph_layer]
else:
to_keep = None
sim_mat = self.graph_layer.get_affinity(input,
to_keep=to_keep,
nosum=nosum)
return sim_mat
def printGraphGrad(self):
grad_rel = self.graph_layers[0].graph_layer.weight.grad
print torch.min(grad_rel).data.cpu().numpy(), torch.max(
grad_rel).data.cpu().numpy()
def out_f(self, input):
# is_cuda = next(self.parameters()).is_cuda
# if is_cuda:
# input = input.cuda()
identity = False
method = None
feature_out = self.linear_layer(input)
to_keep = self.sparsify
out_graph = self.graph_layer(input,
feature_out,
to_keep=to_keep,
graph_sum=self.graph_sum,
identity=identity,
method=method)
if self.graph_sum:
[out_graph, graph_sum] = out_graph
return out_graph
def out_f_f(self, input):
# is_cuda = next(self.parameters()).is_cuda
# if is_cuda:
# input = input.cuda()
identity = False
method = None
feature_out = self.linear_layer(input)
# to_keep = self.sparsify
# out_graph = self.graph_layer(input, feature_out, to_keep = to_keep, graph_sum = self.graph_sum, identity = identity, method = method)
# if self.graph_sum:
# [out_graph, graph_sum] = out_graph
return feature_out
def __init__(self,
n_classes,
deno,
in_out=None,
feat_dim=None,
graph_size=None,
method='cos',
sparsify=0.5,
non_lin='RL',
aft_nonlin='RL',
sigmoid=False,
layer_bef=None,
graph_sum=False,
background=False,
just_graph=False,
feat_ret=False):
super(Graph_Multi_Video, self).__init__()
self.num_classes = n_classes
self.background = background
self.feat_ret = feat_ret
if self.background:
assert sigmoid
n_classes += 1
self.deno = deno
self.graph_size = graph_size
self.sparsify = sparsify
self.graph_sum = graph_sum
self.just_graph = just_graph
if in_out is None:
in_out = [2048, 512]
if feat_dim is None:
feat_dim = [2048, 256]
num_layers = 1
print 'NUM LAYERS', num_layers, in_out
self.bn = None
# nn.BatchNorm1d(2048, affine = False)
self.linear_layer = nn.Linear(feat_dim[0], feat_dim[1], bias=True)
# self.graph_layer = nn.ModuleList()
# self.last_graph = nn.ModuleList()
self.graph_layer = Graph_Layer_Wrapper(in_out[0],
n_out=in_out[1],
non_lin=non_lin,
method=method,
aft_nonlin=aft_nonlin)
last_graph = []
last_graph.append(nn.Dropout(0.5))
last_graph.append(nn.Linear(in_out[-1], n_classes, bias=True))
if sigmoid:
last_graph.append(nn.Sigmoid())
self.last_graph = nn.Sequential(*last_graph)
def __init__(self,
n_classes,
deno,
in_out = None,
feat_dim = None,
in_out_feat = None,
graph_size = None,
method = 'cos',
sparsify = 0.5,
non_lin = 'RL',
aft_nonlin = 'RL',
aft_nonlin_feat = 'RL',
sigmoid = False,
layer_bef = None,
graph_sum = False,
background = False,
just_graph = False
):
super(Graph_Multi_Video, self).__init__()
self.num_classes = n_classes
self.background = background
if self.background:
assert sigmoid
n_classes+=1
self.deno = deno
self.graph_size = graph_size
self.sparsify = sparsify
self.graph_sum = graph_sum
self.just_graph = just_graph
if in_out_feat is None:
in_out_feat = [2048,1024]
if in_out is None:
in_out = [1024,512]
if feat_dim is None:
feat_dim = [1024,256]
assert feat_dim[0]==in_out_feat[1]==in_out[0]
# num_layers = 1
# print 'NUM LAYERS', num_layers, in_out
self.num_branches = 2
print 'self.num_branches', self.num_branches
self.bn =None
# nn.BatchNorm1d(2048, affine = False)
self.feature = []
self.feature.append(nn.Linear(in_out_feat[0], in_out_feat[1], bias = True))
to_pend = aft_nonlin_feat.split('_')
for tp in to_pend:
if tp.lower()=='ht':
self.feature.append(nn.Hardtanh())
elif tp.lower()=='rl':
self.feature.append(nn.ReLU())
elif tp.lower()=='l2':
self.feature.append(Normalize())
elif tp.lower()=='ln':
self.feature.append(nn.LayerNorm(n_out))
elif tp.lower()=='bn':
self.feature.append(nn.BatchNorm1d(n_out, affine = False, track_running_stats = False))
elif tp.lower()=='sig':
self.feature.append(nn.Sigmoid())
else:
error_message = str('non_lin %s not recognized', non_lin)
raise ValueError(error_message)
self.feature = nn.Sequential(*self.feature)
# self.feature_classifier = nn.Linear(in_out[-1],n_classes)
self.linear_layer = nn.Linear(feat_dim[0], feat_dim[1], bias = True)
self.graph_layer = Graph_Layer_Wrapper(in_out[0],n_out = in_out[1], non_lin = non_lin, method = method, aft_nonlin = aft_nonlin)
last_graph = []
last_graph.append(nn.Dropout(0.5))
last_graph.append(nn.Linear(in_out[-1],n_classes))
if sigmoid:
last_graph.append(nn.Sigmoid())
self.last_graph = nn.Sequential(*last_graph)
last_feat = []
last_feat.append(nn.Dropout(0.5))
last_feat.append(nn.Linear(in_out_feat[-1],n_classes))
if sigmoid:
last_feat.append(nn.Sigmoid())
# last_feat.append(nn.Softmax(dim=0))
self.last_feat = nn.Sequential(*last_feat)
def __init__(self,
n_classes,
deno,
in_out=None,
feat_dim=None,
graph_size=None,
method='cos',
sparsify=0.5,
non_lin='RL',
aft_nonlin='RL',
sigmoid=False,
layer_bef=None,
graph_sum=False):
super(Graph_Multi_Video, self).__init__()
self.num_classes = n_classes
# self.background = background
# if self.background:
# assert sigmoid
# n_classes+=1
self.deno = deno
self.graph_size = graph_size
self.sparsify = sparsify
self.graph_sum = graph_sum
# self.just_graph = just_graph
if in_out is None:
in_out = [2048, 512]
if feat_dim is None:
feat_dim = [2048, 256]
num_layers = 1
print 'NUM LAYERS', num_layers, in_out
self.bn = None
# nn.BatchNorm1d(2048, affine = False)
self.linear_layer = nn.Linear(feat_dim[0], feat_dim[1], bias=True)
# self.graph_layer = nn.ModuleList()
# self.last_graph = nn.ModuleList()
self.graph_layer = Graph_Layer_Wrapper(in_out[0],
n_out=in_out[1],
non_lin=non_lin,
method=method,
aft_nonlin=aft_nonlin)
self.det_branch = []
self.class_branch = []
branches = [self.det_branch, self.class_branch]
for branch in branches:
# branch.append(nn.ReLU())
branch.append(nn.Dropout(0.5))
branch.append(nn.Linear(in_out[1], self.num_classes))
[self.det_branch, self.class_branch] = branches
self.det_branch.append(nn.Hardtanh())
self.det_branch_smax = nn.Softmax(dim=0)
self.class_branch.append(nn.Softmax(dim=1))
self.det_branch = nn.Sequential(*self.det_branch)
self.class_branch = nn.Sequential(*self.class_branch)
last_graph = []
last_graph.append(nn.Dropout(0.5))
last_graph.append(nn.Linear(in_out[-1], n_classes))
self.last_graph = nn.Sequential(*last_graph)
def __init__(self,
n_classes,
deno,
pretrained,
in_out = None,
graph_size = None,
method = 'cos',
num_switch = 1,
focus = 0,
sparsify = False,
non_lin = 'HT',
normalize = [True,True]
):
super(Graph_Multi_Video, self).__init__()
self.num_classes = n_classes
self.deno = deno
self.graph_size = graph_size
self.sparsify = sparsify
if in_out is None:
in_out = [2048,64,2048,64]
num_layers = len(in_out)-3
# non_lin = 'HT'
print 'NUM LAYERS', num_layers, in_out
self.linear_layer = nn.Linear(in_out[0], in_out[1], bias = False)
# for param in self.linear_layer.parameters():
# param.requires_grad = False
# non_lin = 'HT'
if pretrained=='ucf':
model_file = '../experiments/just_mill_flexible_deno_8_n_classes_20_layer_sizes_2048_64_ucf/all_classes_False_just_primary_False_limit_500_cw_True_MultiCrossEntropy_100_step_100_0.1_0.001_0.001/model_99.pt'
elif pretrained=='activitynet':
model_file = '../experiments/just_mill_flexible_deno_8_n_classes_100_layer_sizes_2048_64_activitynet/all_classes_False_just_primary_False_limit_500_cw_True_MultiCrossEntropy_50_step_50_0.1_0.001_0.001/model_49.pt'
elif pretrained=='random':
model_file = '../experiments/just_mill_flexible_deno_8_n_classes_20_layer_sizes_2048_64_ucf/all_classes_False_just_primary_False_limit_500_cw_True_MultiCrossEntropy_100_step_100_0.1_0_0.001/model_99.pt'
elif pretrained=='default':
model_file = None
else:
error_message = 'Similarity method %s not valid' % method
raise ValueError(error_message)
if model_file is not None:
model_temp = torch.load(model_file)
self.linear_layer.weight.data = model_temp.linear.weight.data
else:
print 'NO MODEL FILE AAAAAAAA'
self.graph_layers = nn.ModuleList()
for num_layer in range(num_layers):
self.graph_layers.append(Graph_Layer_Wrapper(in_out[num_layer+2],n_out = in_out[num_layer+3], non_lin = non_lin, method = method))
last_linear = []
if non_lin =='HT':
last_linear.append(nn.Hardtanh())
elif non_lin =='RL':
last_linear.append(nn.ReLU())
else:
error_message = str('Non lin %s not valid', non_lin)
raise ValueError(error_message)
if normalize[0]:
last_linear.append(Normalize())
last_linear.append(nn.Dropout(0.5))
last_linear.append(nn.Linear(in_out[1],n_classes))
last_linear = nn.Sequential(*last_linear)
self.last_linear = last_linear
last_graph = []
if non_lin =='HT':
last_graph.append(nn.Hardtanh())
elif non_lin =='RL':
last_graph.append(nn.ReLU())
else:
error_message = str('Non lin %s not valid', non_lin)
raise ValueError(error_message)
if normalize[1]:
last_graph.append(Normalize())
# last_graph.append(nn.Hardtanh())
# last_graph.append(Normalize())
last_graph.append(nn.Dropout(0.5))
last_graph.append(nn.Linear(in_out[-1],n_classes))
last_graph = nn.Sequential(*last_graph)
self.last_graph = last_graph
if type(num_switch)==type(1):
num_switch = [num_switch,num_switch]
self.num_switch = num_switch
self.epoch_counters = [0,0]
self.focus = focus
self.epoch_last = 0
