def __init__(self, batch_size, train_video_length_info, unit_feature_size, unit_size, lambda_reg, lr, train_clip_path, test_clip_path, train_flow_feature_dir, train_appr_feature_dir, test_flow_feature_dir, test_appr_feature_dir):
self.batch_size = batch_size
self.test_batch_size = 1
self.lr = lr
self.lambda_reg = lambda_reg
self.unit_feature_size = unit_feature_size # 4096
self.visual_feature_dim = unit_feature_size # 4096
self.train_set = TrainingDataSet(train_flow_feature_dir,train_appr_feature_dir,train_clip_path, batch_size, train_video_length_info,unit_feature_size,unit_size)
self.test_set = TestingDataSet(test_flow_feature_dir, test_appr_feature_dir, test_clip_path, self.test_batch_size)
def __init__(self, batch_size, train_csv_path, test_csv_path, test_visual_feature_dir, train_visual_feature_dir):
self.batch_size = batch_size
self.test_batch_size = 1
self.vs_lr = 0.005
self.lambda_regression = 0.01
self.alpha = 1.0/batch_size
self.semantic_size = 1024 # the size of visual and semantic comparison size
self.sentence_embedding_size = 4800
self.visual_feature_dim = 4096*3
self.train_set=TrainingDataSet(train_visual_feature_dir, train_csv_path, self.batch_size)
self.test_set=TestingDataSet(test_visual_feature_dir, test_csv_path, self.test_batch_size)
def __init__(self, config):
"""Initialization
"""
self.config = config
self.sess = None
self.saver = None
self.train_clip_path = self.config.train_clip_path
self.background_path = self.config.background_path
self.test_clip_path = self.config.test_clip_path
self.train_flow_feature_dir = self.config.train_flow_feature_dir
self.train_appr_feature_dir = self.config.train_appr_feature_dir
self.test_flow_feature_dir = self.config.test_flow_feature_dir
self.test_appr_feature_dir = self.config.test_appr_feature_dir
self.test_len_dict = self.config.test_len_dict
self.batch_size = self.config.batch_size
self.test_batch_size = 1
self.middle_layer_size = 1000
self.lambda_reg = float(self.config.lambda_reg)
self.action_class_num = self.config.action_class_num
self.feat_type = self.config.feat_type
self.visual_feature_dim = self.config.visual_feature_dim
# Initialize the training data and testing data
self.train_set = TrainingDataSet(self.config,
self.train_flow_feature_dir,
self.train_appr_feature_dir,
self.train_clip_path,
self.background_path)
self.test_set = TestingDataSet(self.config, self.test_flow_feature_dir,
self.test_appr_feature_dir,
self.test_clip_path,
self.test_batch_size,
self.test_len_dict)
# Path to save the summary of the models
self.summary_dir = os.path.join('./summary', self.config.save_name)
if not os.path.exists(self.summary_dir):
os.mkdir(self.summary_dir)
if self.config.issave == 'Yes':
self.model_dir = os.path.join('./model', self.config.save_name)
if not os.path.exists(self.model_dir):
os.mkdir(self.model_dir)
def __init__(self, batch_size, train_video_length_info, ctx_num,
unit_feature_size, unit_size, lambda_reg, lr, train_clip_path,
background_path, test_clip_path, train_visual_feature_dir,
test_visual_feature_dir):
self.batch_size = batch_size
self.test_batch_size = 1
self.lr = lr
self.lambda_reg = lambda_reg
self.unit_feature_size = unit_feature_size
self.visual_feature_dim = unit_feature_size * 3
self.train_set = TrainingDataSet(train_visual_feature_dir,
train_clip_path, background_path,
batch_size, train_video_length_info,
ctx_num, unit_feature_size, unit_size)
self.test_set = TestingDataSet(test_visual_feature_dir, test_clip_path,
self.test_batch_size, ctx_num)
def __init__(self, batch_size, pool_size, train_csv_path, test_csv_path,
test_visual_feature_dir, train_visual_feature_dir):
self.batch_size = batch_size
self.test_batch_size = 1
self.vs_lr = 0.0001
self.lambda_regression = 0.01
self.alpha = 1.0 / batch_size
#self.alpha=0.06
self.pool_size = pool_size
self.semantic_size = 1024
self.sentence_embedding_size = 4800
self.visual_feature_dim = 4096
self.train_set = TrainingDataSet(train_visual_feature_dir,
train_csv_path, self.batch_size)
self.test_set = TestingDataSet(test_visual_feature_dir, test_csv_path,
self.test_batch_size)
self.context_num = 1
def __init__(self, batch_size,train_video_length_info,ctx_num,unit_feature_size,unit_size,lambda_reg,lr,train_clip_path,background_path,test_clip_path,train_visual_feature_dir,test_visual_feature_dir):
self.batch_size = batch_size
self.test_batch_size=1
self.lr=lr
self.lambda_reg=lambda_reg
self.unit_feature_size=unit_feature_size
self.visual_feature_dim=unit_feature_size*3
self.train_set=TrainingDataSet(train_visual_feature_dir,train_clip_path,background_path,batch_size, train_video_length_info,ctx_num,unit_feature_size,unit_size)
self.test_set=TestingDataSet(test_visual_feature_dir,test_clip_path,self.test_batch_size,ctx_num)
def __init__(self, batch_size, ctx_num, unit_size, unit_feature_size,
action_class_num, lr, lambda_reg, train_clip_path,
background_path, test_clip_path, train_flow_feature_dir,
train_appr_feature_dir, test_flow_feature_dir,
test_appr_feature_dir):
self.batch_size = batch_size
self.test_batch_size = 1 # 测试的时候batch = 1个clip,训练的时候batcch = 128
self.middle_layer_size = 1000
self.vs_lr = lr
self.lambda_reg = lambda_reg # 1.0
self.action_class_num = action_class_num # 20
self.visual_feature_dim = unit_feature_size * 3 # 4096*3
self.train_set = TrainingDataSet(train_flow_feature_dir,
train_appr_feature_dir,
train_clip_path, background_path,
batch_size, ctx_num, unit_size,
unit_feature_size, action_class_num)
self.test_set = TestingDataSet(test_flow_feature_dir,
test_appr_feature_dir, test_clip_path,
self.test_batch_size, unit_size)
def __init__(self, batch_size, train_csv_path, test_csv_path, test_visual_feature_dir, train_visual_feature_dir):
self.batch_size = batch_size
self.test_batch_size = 1
self.vs_lr = 0.005
self.lambda_regression = 0.01
self.alpha = 1.0/batch_size
self.semantic_size = 1024 # the size of visual and semantic comparison size
self.sentence_embedding_size = 4800
self.visual_feature_dim = 4096*3
self.train_set=TrainingDataSet(train_visual_feature_dir, train_csv_path, self.batch_size)
self.test_set=TestingDataSet(test_visual_feature_dir, test_csv_path, self.test_batch_size)
def __init__(self, batch_size, train_csv_path, test_csv_path,
test_visual_feature_dir, sliding_dir,
sliding_training_sample_file, test_clip_sentence_pairs_path,
test_swin_txt_path, train_softmax_dir, test_softmax_dir):
self.batch_size = batch_size
self.test_batch_size = 1
self.vs_lr = 0.005
self.lambda_regression = 0.01
self.alpha = 1.0 / batch_size
self.semantic_size = 1024 # the size of visual and semantic comparison size
self.action_semantic_size = 300
self.sentence_embedding_size = 4800
self.visual_feature_dim = 4096 * 3
self.train_set = TrainingDataSet(sliding_dir,
sliding_training_sample_file,
train_csv_path, batch_size,
train_softmax_dir)
self.test_set = TestingDataSet(test_visual_feature_dir, test_csv_path,
self.test_batch_size,
test_swin_txt_path, test_softmax_dir,
test_clip_sentence_pairs_path)
class CTRL_Model(object):
def __init__(self, batch_size, train_csv_path, test_csv_path, test_visual_feature_dir, train_visual_feature_dir):
self.batch_size = batch_size
self.test_batch_size = 1
self.vs_lr = 0.005
self.lambda_regression = 0.01
self.alpha = 1.0/batch_size
self.semantic_size = 1024 # the size of visual and semantic comparison size
self.sentence_embedding_size = 4800
self.visual_feature_dim = 4096*3
self.train_set=TrainingDataSet(train_visual_feature_dir, train_csv_path, self.batch_size)
self.test_set=TestingDataSet(test_visual_feature_dir, test_csv_path, self.test_batch_size)
'''
used in training alignment model, CTRL(aln)
'''
def fill_feed_dict_train(self):
image_batch,sentence_batch,offset_batch = self.train_set.next_batch()
input_feed = {
self.visual_featmap_ph_train: image_batch,
self.sentence_ph_train: sentence_batch,
self.offset_ph: offset_batch
}
return input_feed
'''
used in training alignment+regression model, CTRL(reg)
'''
def fill_feed_dict_train_reg(self):
image_batch, sentence_batch, offset_batch = self.train_set.next_batch_iou()
input_feed = {
self.visual_featmap_ph_train: image_batch,
self.sentence_ph_train: sentence_batch,
self.offset_ph: offset_batch
}
return input_feed
'''
cross modal processing module
'''
def cross_modal_comb(self, visual_feat, sentence_embed, batch_size):
vv_feature = tf.reshape(tf.tile(visual_feat, [batch_size, 1]),
[batch_size, batch_size, self.semantic_size])
ss_feature = tf.reshape(tf.tile(sentence_embed,[1, batch_size]),[batch_size, batch_size, self.semantic_size])
concat_feature = tf.reshape(tf.concat(2,[vv_feature, ss_feature]),[batch_size, batch_size, self.semantic_size+self.semantic_size])
print concat_feature.get_shape().as_list()
mul_feature = tf.mul(vv_feature, ss_feature)
add_feature = tf.add(vv_feature, ss_feature)
comb_feature = tf.reshape(tf.concat(2, [mul_feature, add_feature, concat_feature]),[1, batch_size, batch_size, self.semantic_size*4])
return comb_feature
'''
visual semantic inference, including visual semantic alignment and clip location regression
'''
def visual_semantic_infer(self, visual_feature_train, sentence_embed_train, visual_feature_test, sentence_embed_test):
name="CTRL_Model"
with tf.variable_scope(name):
print "Building training network...............................\n"
transformed_clip_train = fc('v2s_lt', visual_feature_train, output_dim=self.semantic_size)
transformed_clip_train_norm = tf.nn.l2_normalize(transformed_clip_train, dim=1)
transformed_sentence_train = fc('s2s_lt', sentence_embed_train, output_dim=self.semantic_size)
transformed_sentence_train_norm = tf.nn.l2_normalize(transformed_sentence_train, dim=1)
cross_modal_vec_train = self.cross_modal_comb(transformed_clip_train_norm, transformed_sentence_train_norm, self.batch_size)
sim_score_mat_train = vs_multilayer.vs_multilayer(cross_modal_vec_train, "vs_multilayer_lt", middle_layer_dim=1000)
sim_score_mat_train = tf.reshape(sim_score_mat_train,[self.batch_size, self.batch_size, 3])
tf.get_variable_scope().reuse_variables()
print "Building test network...............................\n"
transformed_clip_test = fc('v2s_lt', visual_feature_test, output_dim=self.semantic_size)
transformed_clip_test_norm = tf.nn.l2_normalize(transformed_clip_test, dim=1)
transformed_sentence_test = fc('s2s_lt', sentence_embed_test, output_dim=self.semantic_size)
transformed_sentence_test_norm = tf.nn.l2_normalize(transformed_sentence_test, dim=1)
cross_modal_vec_test = self.cross_modal_comb(transformed_clip_test_norm, transformed_sentence_test_norm, self.test_batch_size)
sim_score_mat_test = vs_multilayer.vs_multilayer(cross_modal_vec_test, "vs_multilayer_lt", reuse=True, middle_layer_dim=1000)
sim_score_mat_test = tf.reshape(sim_score_mat_test, [3])
return sim_score_mat_train, sim_score_mat_test
'''
compute alignment and regression loss
'''
def compute_loss_reg(self, sim_reg_mat, offset_label):
sim_score_mat, p_reg_mat, l_reg_mat = tf.split(2, 3, sim_reg_mat)
sim_score_mat = tf.reshape(sim_score_mat, [self.batch_size, self.batch_size])
l_reg_mat = tf.reshape(l_reg_mat, [self.batch_size, self.batch_size])
p_reg_mat = tf.reshape(p_reg_mat, [self.batch_size, self.batch_size])
# unit matrix with -2
I_2 = tf.diag(tf.constant(-2.0, shape=[self.batch_size]))
all1 = tf.constant(1.0, shape=[self.batch_size, self.batch_size])
# | -1 1 1... |
# mask_mat = | 1 -1 -1... |
# | 1 1 -1 ... |
mask_mat = tf.add(I_2, all1)
# loss cls, not considering iou
I = tf.diag(tf.constant(1.0, shape=[self.batch_size]))
I_half = tf.diag(tf.constant(0.5, shape=[self.batch_size]))
batch_para_mat = tf.constant(self.alpha, shape=[self.batch_size, self.batch_size])
para_mat = tf.add(I,batch_para_mat)
loss_mat = tf.log(tf.add(all1, tf.exp(tf.mul(mask_mat, sim_score_mat))))
loss_mat = tf.mul(loss_mat, para_mat)
loss_align = tf.reduce_mean(loss_mat)
# regression loss
l_reg_diag = tf.matmul(tf.mul(l_reg_mat, I), tf.constant(1.0, shape=[self.batch_size, 1]))
p_reg_diag = tf.matmul(tf.mul(p_reg_mat, I), tf.constant(1.0, shape=[self.batch_size, 1]))
offset_pred = tf.concat(1, (p_reg_diag, l_reg_diag))
loss_reg = tf.reduce_mean(tf.abs(tf.sub(offset_pred, offset_label)))
loss=tf.add(tf.mul(self.lambda_regression, loss_reg), loss_align)
return loss, offset_pred, loss_reg
def init_placeholder(self):
visual_featmap_ph_train = tf.placeholder(tf.float32, shape=(self.batch_size, self.visual_feature_dim))
sentence_ph_train = tf.placeholder(tf.float32, shape=(self.batch_size, self.sentence_embedding_size))
offset_ph = tf.placeholder(tf.float32, shape=(self.batch_size,2))
visual_featmap_ph_test = tf.placeholder(tf.float32, shape=(self.test_batch_size, self.visual_feature_dim))
sentence_ph_test = tf.placeholder(tf.float32, shape=(self.test_batch_size, self.sentence_embedding_size))
return visual_featmap_ph_train,sentence_ph_train,offset_ph,visual_featmap_ph_test,sentence_ph_test
def get_variables_by_name(self,name_list):
v_list = tf.trainable_variables()
v_dict = {}
for name in name_list:
v_dict[name] = []
for v in v_list:
for name in name_list:
if name in v.name: v_dict[name].append(v)
for name in name_list:
print "Variables of <"+name+">"
for v in v_dict[name]:
print " "+v.name
return v_dict
def training(self, loss):
v_dict = self.get_variables_by_name(["lt"])
vs_optimizer = tf.train.AdamOptimizer(self.vs_lr, name='vs_adam')
vs_train_op = vs_optimizer.minimize(loss, var_list=v_dict["lt"])
return vs_train_op
def construct_model(self):
# initialize the placeholder
self.visual_featmap_ph_train, self.sentence_ph_train, self.offset_ph, self.visual_featmap_ph_test, self.sentence_ph_test=self.init_placeholder()
# build inference network
sim_reg_mat, sim_reg_mat_test = self.visual_semantic_infer(self.visual_featmap_ph_train, self.sentence_ph_train, self.visual_featmap_ph_test, self.sentence_ph_test)
# compute loss
self.loss_align_reg, offset_pred, loss_reg = self.compute_loss_reg(sim_reg_mat, self.offset_ph)
# optimize
self.vs_train_op = self.training(self.loss_align_reg)
return self.loss_align_reg, self.vs_train_op, sim_reg_mat_test, offset_pred, loss_reg
class CBR_Model(object):
def __init__(self, batch_size, ctx_num, unit_size, unit_feature_size,
action_class_num, lr, lambda_reg, train_clip_path,
background_path, test_clip_path, train_flow_feature_dir,
train_appr_feature_dir, test_flow_feature_dir,
test_appr_feature_dir):
self.batch_size = batch_size
self.test_batch_size = 1 # 测试的时候batch = 1个clip,训练的时候batcch = 128
self.middle_layer_size = 1000
self.vs_lr = lr
self.lambda_reg = lambda_reg # 1.0
self.action_class_num = action_class_num # 20
self.visual_feature_dim = unit_feature_size * 3 # 4096*3
self.train_set = TrainingDataSet(train_flow_feature_dir,
train_appr_feature_dir,
train_clip_path, background_path,
batch_size, ctx_num, unit_size,
unit_feature_size, action_class_num)
self.test_set = TestingDataSet(test_flow_feature_dir,
test_appr_feature_dir, test_clip_path,
self.test_batch_size, unit_size)
def fill_feed_dict_train(self):
image_batch, label_batch, offset_batch, one_hot_label_batch = self.train_set.next_batch(
)
input_feed = {
self.visual_featmap_ph_train: image_batch, # 加载的inter特征和上下文特征
self.label_ph: label_batch,
self.offset_ph: offset_batch,
self.one_hot_label_ph: one_hot_label_batch
}
return input_feed
# 计算总的loss和回归loss
# vs_multilayer由两个全连接层构成，将该clip对应的特征输入，生成用于分类和回归的特征
def compute_loss_reg(self, visual_feature, offsets, labels,
one_hot_labels):
cls_reg_vec = vs_multilayer.vs_multilayer(
visual_feature,
"CBR",
middle_layer_dim=self.middle_layer_size,
output_layer_dim=(self.action_class_num + 1) * 3)
cls_reg_vec = tf.reshape(
cls_reg_vec, [self.batch_size,
(self.action_class_num + 1) * 3]) # [128,21*3]
cls_score_vec = cls_reg_vec[:, :self.action_class_num + 1]
start_offset_pred = cls_reg_vec[:, self.action_class_num +
1:(self.action_class_num + 1) * 2]
end_offset_pred = cls_reg_vec[:, (self.action_class_num + 1) * 2:]
#classification loss
loss_cls_vec = tf.nn.sparse_softmax_cross_entropy_with_logits(
cls_score_vec, labels)
loss_cls = tf.reduce_mean(loss_cls_vec)
# regression loss
pick_start_offset_pred = []
pick_end_offset_pred = []
# 选取第K个sampel的属于某个类别的回归值。参考论文中的回归计算
for k in range(self.batch_size): # 选取第K个sample的回归预测值
pick_start_offset_pred.append(start_offset_pred[k, labels[k]])
pick_end_offset_pred.append(end_offset_pred[k, labels[k]])
pick_start_offset_pred = tf.reshape(tf.stack(pick_start_offset_pred),
[self.batch_size, 1])
pick_end_offset_pred = tf.reshape(tf.stack(pick_end_offset_pred),
[self.batch_size, 1])
labels_1 = tf.to_float(tf.not_equal(
labels, 0)) # 选取对应的类别的回归值,labels中保存的是该sample属于哪个类别
label_tmp = tf.to_float(tf.reshape(labels_1, [self.batch_size, 1]))
label_for_reg = tf.concat(1, [label_tmp, label_tmp]) # 按列进行拼接 [128,2]
offset_pred = tf.concat(
1, (pick_start_offset_pred, pick_end_offset_pred)) # [128,2]
loss_reg = tf.reduce_mean(
tf.mul(tf.abs(tf.sub(offset_pred, offsets)), label_for_reg))
loss = tf.add(tf.mul(self.lambda_reg, loss_reg), loss_cls)
return loss, loss_reg
# 创建placeholder
def init_placeholder(self):
visual_featmap_ph_train = tf.placeholder(
tf.float32,
shape=(self.batch_size, self.visual_feature_dim)) # (128,12288)
label_ph = tf.placeholder(tf.int32, shape=(self.batch_size)) # (128,)
offset_ph = tf.placeholder(tf.float32,
shape=(self.batch_size, 2)) # (128,2)
one_hot_label_ph = tf.placeholder(
tf.float32,
shape=(self.batch_size, self.action_class_num + 1)) # (128,21)
visual_featmap_ph_test = tf.placeholder(
tf.float32,
shape=(self.test_batch_size, self.visual_feature_dim)) # (1,12288)
return visual_featmap_ph_train, visual_featmap_ph_test, label_ph, offset_ph, one_hot_label_ph
# 测试，输出结果
def eval(self, visual_feature_test):
sim_score = vs_multilayer.vs_multilayer(
visual_feature_test,
"CBR",
middle_layer_dim=self.middle_layer_size,
output_layer_dim=(self.action_class_num + 1) * 3,
dropout=False,
reuse=True)
sim_score = tf.reshape(sim_score, [(self.action_class_num + 1) * 3])
return sim_score
def get_variables_by_name(self, name_list): # name_list : ['CBR']
v_list = tf.trainable_variables() #tf.trainable_variables返回的是需要训练的变量列表
v_dict = {}
for name in name_list:
v_dict[name] = []
# 遍历名为name的变量列表，将其添加到v_dict
for v in v_list:
for name in name_list:
if name in v.name: v_dict[name].append(v)
for name in name_list:
print "Variables of <" + name + ">"
for v in v_dict[name]:
print " " + v.name
return v_dict
# 训练，现货区可训练变量，再构造优化器，最终最小化loss
def training(self, loss):
v_dict = self.get_variables_by_name(["CBR"]) # 获取可训练变量列表
vs_optimizer = tf.train.AdamOptimizer(self.vs_lr, name='vs_adam')
vs_train_op = vs_optimizer.minimize(loss, var_list=v_dict["CBR"])
return vs_train_op
def construct_model(self):
#construct the network:
self.visual_featmap_ph_train, self.visual_featmap_ph_test, self.label_ph, self.offset_ph, self.one_hot_label_ph = self.init_placeholder(
)
visual_featmap_ph_train_norm = tf.nn.l2_normalize(
self.visual_featmap_ph_train, dim=1) # 按行进行L2归一化
visual_featmap_ph_test_norm = tf.nn.l2_normalize(
self.visual_featmap_ph_test, dim=1)
# 计算分类loss和回归Loss，返回最终的总的loss和回归loss
self.loss, loss_reg = self.compute_loss_reg(
visual_featmap_ph_train_norm, self.offset_ph, self.label_ph,
self.one_hot_label_ph)
self.vs_train_op = self.training(self.loss)
vs_eval_op = self.eval(visual_featmap_ph_test_norm)
return self.loss, self.vs_train_op, vs_eval_op, loss_reg #返回总的loss，训练操作，测试操作，
class PATE_Model(object):
def __init__(self, batch_size, train_video_length_info, unit_feature_size,
unit_size, lambda_reg, lr, train_clip_path, test_clip_path,
train_flow_feature_dir, train_appr_feature_dir,
test_flow_feature_dir, test_appr_feature_dir):
self.batch_size = batch_size
self.test_batch_size = 1
self.lr = lr
self.lambda_reg = lambda_reg
self.unit_feature_size = unit_feature_size
self.visual_feature_dim = unit_feature_size
self.train_set = TrainingDataSet(train_flow_feature_dir,
train_appr_feature_dir,
train_clip_path, batch_size,
train_video_length_info,
unit_feature_size, unit_size)
self.test_set = TestingDataSet(test_flow_feature_dir,
test_appr_feature_dir, test_clip_path,
self.test_batch_size)
def fill_feed_dict_train_reg(self):
image_batch, label_batch = self.train_set.next_batch()
input_feed = {
self.visual_featmap_ph_train: image_batch,
self.label_ph: label_batch
}
return input_feed
# construct the top network and compute loss
def compute_loss(self, visual_feature, labels):
cls_vec = vs_multilayer.vs_multilayer(visual_feature,
"PATE",
middle_layer_dim=1000)
cls_vec = tf.reshape(cls_vec, [self.batch_size, 2])
#classification loss
loss_cls_vec = tf.nn.sparse_softmax_cross_entropy_with_logits(
cls_vec, labels)
loss_cls = tf.reduce_mean(loss_cls_vec)
return loss_cls
def init_placeholder(self):
visual_featmap_ph_train = tf.placeholder(
tf.float32, shape=(self.batch_size, self.visual_feature_dim))
label_ph = tf.placeholder(tf.int32, shape=(self.batch_size))
visual_featmap_ph_test = tf.placeholder(
tf.float32, shape=(self.test_batch_size, self.visual_feature_dim))
return visual_featmap_ph_train, visual_featmap_ph_test, label_ph
# set up the eval op
def eval(self, visual_feature_test):
outputs = vs_multilayer.vs_multilayer(visual_feature_test,
"PATE",
middle_layer_dim=1000,
reuse=True)
outputs = tf.reshape(outputs, [2])
return outputs
# return all the variables that contains the name in name_list
def get_variables_by_name(self, name_list):
v_list = tf.trainable_variables()
v_dict = {}
for name in name_list:
v_dict[name] = []
for v in v_list:
for name in name_list:
if name in v.name: v_dict[name].append(v)
for name in name_list:
print "Variables of <" + name + ">"
for v in v_dict[name]:
print " " + v.name
return v_dict
# set up the optimizer
def training(self, loss):
v_dict = self.get_variables_by_name(["PATE"])
vs_optimizer = tf.train.AdamOptimizer(self.lr, name='vs_adam')
vs_train_op = vs_optimizer.minimize(loss, var_list=v_dict["PATE"])
return vs_train_op
# construct the network
def construct_model(self):
self.visual_featmap_ph_train, self.visual_featmap_ph_test, self.label_ph = self.init_placeholder(
)
visual_featmap_ph_train_norm = tf.nn.l2_normalize(
self.visual_featmap_ph_train, dim=1)
visual_featmap_ph_test_norm = tf.nn.l2_normalize(
self.visual_featmap_ph_test, dim=1)
self.loss_cls = self.compute_loss(visual_featmap_ph_train_norm,
self.label_ph)
self.train_op = self.training(self.loss_cls)
eval_op = self.eval(visual_featmap_ph_test_norm)
return self.loss_cls, self.train_op, eval_op
def __init__(self, ):
self.train_set = TrainingDataSet(self.batch_size)
self.val_set = ValidationDataSet()
def __init__(self, ):
self.train_set = TrainingDataSet(self.batch_size)
self.test_set = TestingDataSet()
class TURN_Model(object):
def __init__(self, batch_size, train_video_length_info, ctx_num,
unit_feature_size, unit_size, lambda_reg, lr, train_clip_path,
background_path, test_clip_path, train_visual_feature_dir,
test_visual_feature_dir):
self.batch_size = batch_size # 128
self.test_batch_size = 1
self.lr = lr
self.lambda_reg = lambda_reg # 2.0
self.unit_feature_size = unit_feature_size # 2048
self.visual_feature_dim = unit_feature_size * 3 # 2048*3 = 6144
#主要是来加载对应clip的特征
self.train_set = TrainingDataSet(train_visual_feature_dir,
train_clip_path, background_path,
batch_size, train_video_length_info,
ctx_num, unit_feature_size, unit_size)
self.test_set = TestingDataSet(test_visual_feature_dir, test_clip_path,
self.test_batch_size, ctx_num)
# 该函数主要加载训练所需的数据
# image_batch中保存了一个batch(128个clip)的特征数据
# label_batch:一个batch数据的标签
# offset_batch：预先计算的真实坐标偏移值
def fill_feed_dict_train_reg(self):
image_batch, label_batch, offset_batch = self.train_set.next_batch()
input_feed = {
self.visual_featmap_ph_train: image_batch,
self.label_ph: label_batch,
self.offset_ph: offset_batch # 真实偏移
}
return input_feed # 返回一个字典，里面包含一个批次的特征，标签和坐标偏移
# construct the top network and compute loss
def compute_loss_reg(self, visual_feature, offsets, labels):
cls_reg_vec = vs_multilayer.vs_multilayer(visual_feature,
"APN",
middle_layer_dim=1000)
cls_reg_vec = tf.reshape(cls_reg_vec, [self.batch_size, 4]) # [128,4]
"""
cls_score_vec_0 : (128,1)
cls_score_vec_1 : (128,1)
p_reg_vec : (128,1)
l_reg_vec : (128,1)
"""
# 将分类和回归向量拆分和组合
cls_score_vec_0, cls_score_vec_1, p_reg_vec, l_reg_vec = tf.split(
1, 4, cls_reg_vec)
cls_score_vec = tf.concat(1, (cls_score_vec_0, cls_score_vec_1))
offset_pred = tf.concat(1, (p_reg_vec, l_reg_vec))
#classification loss
loss_cls_vec = tf.nn.sparse_softmax_cross_entropy_with_logits(
cls_score_vec, labels)
loss_cls = tf.reduce_mean(loss_cls_vec)
# regression loss
label_tmp = tf.to_float(tf.reshape(labels, [self.batch_size, 1]))
label_for_reg = tf.concat(1, [label_tmp, label_tmp])
# offset_pred为最后全连接层的预测的坐标偏移值
# offsets是真实的坐标偏移值
loss_reg = tf.reduce_mean(
tf.mul(tf.abs(tf.sub(offset_pred, offsets)), label_for_reg))
loss = tf.add(tf.mul(self.lambda_reg, loss_reg), loss_cls)
return loss, offset_pred, loss_reg
def init_placeholder(self):
visual_featmap_ph_train = tf.placeholder(
tf.float32, shape=(self.batch_size, self.visual_feature_dim))
label_ph = tf.placeholder(tf.int32, shape=(self.batch_size))
offset_ph = tf.placeholder(tf.float32, shape=(self.batch_size, 2))
visual_featmap_ph_test = tf.placeholder(
tf.float32, shape=(self.test_batch_size, self.visual_feature_dim))
return visual_featmap_ph_train, visual_featmap_ph_test, label_ph, offset_ph
# set up the eval op
def eval(self, visual_feature_test):
#visual_feature_test=tf.reshape(visual_feature_test,[1,4096])
outputs = vs_multilayer.vs_multilayer(visual_feature_test,
"APN",
middle_layer_dim=1000,
reuse=True)
outputs = tf.reshape(outputs, [4])
return outputs
# return all the variables that contains the name in name_list
def get_variables_by_name(self, name_list):
v_list = tf.trainable_variables()
v_dict = {}
for name in name_list:
v_dict[name] = []
for v in v_list:
for name in name_list:
if name in v.name: v_dict[name].append(v)
for name in name_list:
print "Variables of <" + name + ">"
for v in v_dict[name]:
print " " + v.name
return v_dict
# set up the optimizer
def training(self, loss):
v_dict = self.get_variables_by_name(["APN"]) #获取可训练参数
vs_optimizer = tf.train.AdamOptimizer(self.lr, name='vs_adam')
vs_train_op = vs_optimizer.minimize(loss, var_list=v_dict["APN"])
return vs_train_op
# construct the network
def construct_model(self):
"""
self.visual_featmap_ph_train : (128,6144)
self.visual_featmap_ph_test : (1,6144)
self.label_ph : (128,1)
self.offset_ph : (128,2)
"""
self.visual_featmap_ph_train, self.visual_featmap_ph_test, self.label_ph, self.offset_ph = self.init_placeholder(
)
visual_featmap_ph_train_norm = tf.nn.l2_normalize(
self.visual_featmap_ph_train, dim=1)
visual_featmap_ph_test_norm = tf.nn.l2_normalize(
self.visual_featmap_ph_test, dim=1)
# 根据label计算loss,reg_loss,回归值
self.loss_cls_reg, offset_pred, loss_reg = self.compute_loss_reg(
visual_featmap_ph_train_norm, self.offset_ph, self.label_ph)
self.train_op = self.training(self.loss_cls_reg) # 训练操作主要是为了最小化总loss
eval_op = self.eval(visual_featmap_ph_test_norm) # 获取fc层的输出结果用来后续的测试
return self.loss_cls_reg, self.train_op, eval_op, loss_reg
initial_steps = 0
max_steps = 20000
batch_size = 64
train_csv_path = "/home/wam/Action_Recognition/TACoS/train_clip-sentvec.pkl"
test_csv_path = "/home/wam/Action_Recognition/TACoS/test_clip-sentvec.pkl"
test_feature_dir="/home/wam/Action_Recognition/Interval128_256_overlap0.8_c3d_fc6/"
train_feature_dir = "/home/wam/Action_Recognition/Interval64_128_256_512_overlap0.8_c3d_fc6/"
test_batch_size = 1
vs_lr = 0.001
lambda_regression = 0.01
alpha = 1.0/batch_size
semantic_size = 1024 # the size of visual and semantic comparison size
sentence_embedding_size = 4800
visual_feature_dim = 4096*3
train_set=TrainingDataSet(train_feature_dir, train_csv_path, batch_size)
test_set=TestingDataSet(test_feature_dir, test_csv_path, test_batch_size)
def compute_loss_reg(sim_reg_mat, offset_label):
sim_score_mat, p_reg_mat, l_reg_mat = tf.split(sim_reg_mat, 3, 2)
sim_score_mat = tf.reshape(sim_score_mat, [batch_size, batch_size])
l_reg_mat = tf.reshape(l_reg_mat, [batch_size, batch_size])
p_reg_mat = tf.reshape(p_reg_mat, [batch_size, batch_size])
# unit matrix with -2
I_2 = tf.diag(tf.constant(-2.0, shape=[batch_size]))
all1 = tf.constant(1.0, shape=[batch_size, batch_size])
mask_mat = tf.add(I_2, all1)
# loss cls, not considering iou
I = tf.diag(tf.constant(1.0, shape=[batch_size]))
I_half = tf.diag(tf.constant(0.5, shape=[batch_size]))
batch_para_mat = tf.constant(alpha, shape=[batch_size, batch_size])
class TURN_Model(object):
def __init__(self, batch_size,train_video_length_info,ctx_num,unit_feature_size,unit_size,lambda_reg,lr,train_clip_path,background_path,test_clip_path,train_visual_feature_dir,test_visual_feature_dir):
self.batch_size = batch_size
self.test_batch_size=1
self.lr=lr
self.lambda_reg=lambda_reg
self.unit_feature_size=unit_feature_size
self.visual_feature_dim=unit_feature_size*3
self.train_set=TrainingDataSet(train_visual_feature_dir,train_clip_path,background_path,batch_size, train_video_length_info,ctx_num,unit_feature_size,unit_size)
self.test_set=TestingDataSet(test_visual_feature_dir,test_clip_path,self.test_batch_size,ctx_num)
def fill_feed_dict_train_reg(self):
image_batch,label_batch,offset_batch=self.train_set.next_batch()
input_feed = {
self.visual_featmap_ph_train: image_batch,
self.label_ph: label_batch,
self.offset_ph: offset_batch
}
return input_feed
# construct the top network and compute loss
def compute_loss_reg(self,visual_feature,offsets,labels):
cls_reg_vec=vs_multilayer.vs_multilayer(visual_feature,"APN",middle_layer_dim=1000)
cls_reg_vec=tf.reshape(cls_reg_vec,[self.batch_size,4])
cls_score_vec_0,cls_score_vec_1,p_reg_vec,l_reg_vec=tf.split(1,4,cls_reg_vec)
cls_score_vec=tf.concat(1,(cls_score_vec_0,cls_score_vec_1))
offset_pred=tf.concat(1,(p_reg_vec,l_reg_vec))
#classification loss
loss_cls_vec=tf.nn.sparse_softmax_cross_entropy_with_logits(cls_score_vec, labels)
loss_cls=tf.reduce_mean(loss_cls_vec)
# regression loss
label_tmp=tf.to_float(tf.reshape(labels,[self.batch_size,1]))
label_for_reg=tf.concat(1,[label_tmp,label_tmp])
loss_reg=tf.reduce_mean(tf.mul(tf.abs(tf.sub(offset_pred,offsets)),label_for_reg))
loss=tf.add(tf.mul(self.lambda_reg,loss_reg),loss_cls)
return loss,offset_pred,loss_reg
def init_placeholder(self):
visual_featmap_ph_train=tf.placeholder(tf.float32, shape=(self.batch_size,self.visual_feature_dim))
label_ph=tf.placeholder(tf.int32, shape=(self.batch_size))
offset_ph=tf.placeholder(tf.float32, shape=(self.batch_size,2))
visual_featmap_ph_test=tf.placeholder(tf.float32, shape=(self.test_batch_size,self.visual_feature_dim))
return visual_featmap_ph_train,visual_featmap_ph_test,label_ph,offset_ph
# set up the eval op
def eval(self,visual_feature_test):
#visual_feature_test=tf.reshape(visual_feature_test,[1,4096])
outputs=vs_multilayer.vs_multilayer(visual_feature_test,"APN",middle_layer_dim=1000,reuse=True)
outputs=tf.reshape(outputs,[4])
return outputs
# return all the variables that contains the name in name_list
def get_variables_by_name(self,name_list):
v_list=tf.trainable_variables()
v_dict={}
for name in name_list:
v_dict[name]=[]
for v in v_list:
for name in name_list:
if name in v.name: v_dict[name].append(v)
for name in name_list:
print "Variables of <"+name+">"
for v in v_dict[name]:
print " "+v.name
return v_dict
# set up the optimizer
def training(self, loss):
v_dict=self.get_variables_by_name(["APN"])
vs_optimizer=tf.train.AdamOptimizer(self.lr,name='vs_adam')
vs_train_op=vs_optimizer.minimize(loss,var_list=v_dict["APN"])
return vs_train_op
# construct the network
def construct_model(self):
self.visual_featmap_ph_train,self.visual_featmap_ph_test,self.label_ph,self.offset_ph=self.init_placeholder()
visual_featmap_ph_train_norm=tf.nn.l2_normalize(self.visual_featmap_ph_train,dim=1)
visual_featmap_ph_test_norm=tf.nn.l2_normalize(self.visual_featmap_ph_test,dim=1)
self.loss_cls_reg,offset_pred,loss_reg=self.compute_loss_reg(visual_featmap_ph_train_norm,self.offset_ph,self.label_ph)
self.train_op=self.training(self.loss_cls_reg)
eval_op=self.eval(visual_featmap_ph_test_norm)
return self.loss_cls_reg,self.train_op, eval_op,loss_reg
class TURN_Model(object):
ctx_num = 4
unit_size = 16.0
unit_feature_size = 2048
lr = 0.005
lambda_reg = 2.0
batch_size = 128
test_batch_size = 1
visual_feature_dim = unit_feature_size * 3
def __init__(self, ):
self.train_set = TrainingDataSet(self.batch_size)
self.test_set = TestingDataSet()
def fill_feed_dict_train_reg(self):
image_batch, label_batch, offset_batch = self.train_set.next_batch()
image_batch = np.nan_to_num(image_batch)
input_feed = {
self.visual_featmap_ph_train: image_batch,
self.label_ph: label_batch,
self.offset_ph: offset_batch
}
return input_feed
# construct the top network and compute loss
def compute_loss_reg(self, visual_feature, offsets, labels, test=False):
cls_reg_vec = vs_multilayer.vs_multilayer(visual_feature,
"APN",
middle_layer_dim=1000,
test=test)
cls_reg_vec = tf.reshape(cls_reg_vec, [self.batch_size, 4])
cls_score_vec_0, cls_score_vec_1, p_reg_vec, l_reg_vec = tf.split(
cls_reg_vec, 4, 1)
cls_score_vec = tf.concat((cls_score_vec_0, cls_score_vec_1), 1)
offset_pred = tf.concat((p_reg_vec, l_reg_vec), 1)
# classification loss
loss_cls_vec = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=cls_score_vec, labels=labels)
loss_cls = tf.reduce_mean(loss_cls_vec)
# regression loss
label_tmp = tf.to_float(tf.reshape(labels, [self.batch_size, 1]))
label_for_reg = tf.concat([label_tmp, label_tmp], 1)
loss_reg = tf.reduce_mean(
tf.multiply(tf.abs(tf.subtract(offset_pred, offsets)),
label_for_reg))
loss = tf.add(tf.multiply(self.lambda_reg, loss_reg), loss_cls)
return loss, offset_pred, loss_reg
def init_placeholder(self):
visual_featmap_ph_train = tf.placeholder(
tf.float32, shape=(self.batch_size, self.visual_feature_dim))
label_ph = tf.placeholder(tf.int32, shape=(self.batch_size))
offset_ph = tf.placeholder(tf.float32, shape=(self.batch_size, 2))
visual_featmap_ph_test = tf.placeholder(
tf.float32, shape=(self.test_batch_size, self.visual_feature_dim))
# print(visual_featmap_ph_train, visual_featmap_ph_test, label_ph, offset_ph)
return visual_featmap_ph_train, visual_featmap_ph_test, label_ph, offset_ph
# set up the eval op
def eval(self, visual_feature_test):
# visual_feature_test=tf.reshape(visual_feature_test,[1,4096])
outputs = vs_multilayer.vs_multilayer(visual_feature_test,
"APN",
middle_layer_dim=1000,
reuse=True)
outputs = tf.reshape(outputs, [4])
return outputs
# return all the variables that contains the name in name_list
def get_variables_by_name(self, name_list):
v_list = tf.trainable_variables()
v_dict = {}
for name in name_list:
v_dict[name] = []
for v in v_list:
for name in name_list:
if name in v.name: v_dict[name].append(v)
for name in name_list:
print "Variables of <" + name + ">"
for v in v_dict[name]:
print " " + v.name
return v_dict
# set up the optimizer
def training(self, loss):
v_dict = self.get_variables_by_name(["APN"])
vs_optimizer = tf.train.AdamOptimizer(self.lr, name='vs_adam')
vs_train_op = vs_optimizer.minimize(loss, var_list=v_dict["APN"])
return vs_train_op
# construct the network
def construct_model(self, test=False):
self.visual_featmap_ph_train, self.visual_featmap_ph_test, self.label_ph, self.offset_ph = self.init_placeholder(
)
visual_featmap_ph_train_norm = tf.nn.l2_normalize(
self.visual_featmap_ph_train, dim=1)
visual_featmap_ph_test_norm = tf.nn.l2_normalize(
self.visual_featmap_ph_test, dim=1)
self.loss_cls_reg, offset_pred, loss_reg = self.compute_loss_reg(
visual_featmap_ph_train_norm, self.offset_ph, self.label_ph, test)
self.train_op = self.training(self.loss_cls_reg)
eval_op = self.eval(visual_featmap_ph_test_norm)
return self.loss_cls_reg, self.train_op, eval_op, loss_reg
class TAR_Model(object):
def __init__(self, batch_size, train_video_length_info, ctx_num,
central_num, unit_feature_size, unit_size, lambda_reg, lr,
train_clip_path, background_path, test_clip_path,
train_flow_feature_dir, train_appr_feature_dir,
test_flow_feature_dir, test_appr_feature_dir):
self.batch_size = batch_size
self.test_batch_size = 1
self.lr = lr
self.lambda_reg = lambda_reg
self.unit_feature_size = unit_feature_size
self.visual_feature_dim = unit_feature_size
self.train_set = TrainingDataSet(train_flow_feature_dir,
train_appr_feature_dir,
train_clip_path, background_path,
batch_size, train_video_length_info,
ctx_num, central_num,
unit_feature_size, unit_size)
self.test_set = TestingDataSet(test_flow_feature_dir,
test_appr_feature_dir, test_clip_path,
self.test_batch_size, ctx_num)
self.ctx_num = ctx_num
self.central_num = central_num
def fill_feed_dict_train_reg(self):
central_batch, left_batch, right_batch, label_batch, offset_batch = self.train_set.next_batch(
)
input_feed = {
self.central_ph_train: central_batch,
self.left_ph_train: left_batch,
self.right_ph_train: right_batch,
self.label_ph: label_batch,
self.offset_ph: offset_batch
}
return input_feed
# construct the top network and compute loss
def compute_loss_reg(self, central, start, end, offsets, labels):
central_cls, start_reg, end_reg = vs_multilayer.vs_multilayer(
central, start, end, "BLA")
offset_pred = tf.concat(1, (start_reg, end_reg))
#classification loss
loss_cls_vec = tf.nn.sparse_softmax_cross_entropy_with_logits(
central_cls, labels)
loss_cls = tf.reduce_mean(loss_cls_vec)
# regression loss
label_tmp = tf.to_float(tf.reshape(labels, [self.batch_size, 1]))
label_for_reg = tf.concat(1, [label_tmp, label_tmp])
loss_reg = tf.reduce_mean(
tf.mul(tf.abs(tf.sub(offset_pred, offsets)), label_for_reg))
loss = tf.add(tf.mul(self.lambda_reg, loss_reg), loss_cls)
return loss, offset_pred, loss_reg
def init_placeholder(self):
self.central_ph_train = tf.placeholder(tf.float32,
shape=(self.batch_size,
self.central_num,
self.visual_feature_dim))
self.left_ph_train = tf.placeholder(tf.float32,
shape=(self.batch_size,
self.ctx_num,
self.visual_feature_dim))
self.right_ph_train = tf.placeholder(tf.float32,
shape=(self.batch_size,
self.ctx_num,
self.visual_feature_dim))
self.label_ph = tf.placeholder(tf.int32, shape=(self.batch_size))
self.offset_ph = tf.placeholder(tf.float32, shape=(self.batch_size, 2))
self.central_ph_test = tf.placeholder(tf.float32,
shape=(self.test_batch_size,
self.central_num,
self.visual_feature_dim))
self.left_ph_test = tf.placeholder(tf.float32,
shape=(self.test_batch_size,
self.ctx_num,
self.visual_feature_dim))
self.right_ph_test = tf.placeholder(tf.float32,
shape=(self.test_batch_size,
self.ctx_num,
self.visual_feature_dim))
return
# set up the eval op
def eval(self, central, start, end):
central_cls, start_reg, end_reg = vs_multilayer.vs_multilayer(
central, start, end, "BLA", reuse=True)
outputs = tf.concat(1, (central_cls, start_reg, end_reg))
outputs = tf.reshape(outputs, [4])
print "eval output size: " + str(outputs.get_shape().as_list())
return outputs
# return all the variables that contains the name in name_list
def get_variables_by_name(self, name_list):
v_list = tf.trainable_variables()
v_dict = {}
for name in name_list:
v_dict[name] = []
for v in v_list:
for name in name_list:
if name in v.name: v_dict[name].append(v)
for name in name_list:
print "Variables of <" + name + ">"
for v in v_dict[name]:
print " " + v.name
return v_dict
# set up the optimizer
def training(self, loss):
v_dict = self.get_variables_by_name(["BLA"])
vs_optimizer = tf.train.AdamOptimizer(self.lr, name='vs_adam')
vs_train_op = vs_optimizer.minimize(loss, var_list=v_dict["BLA"])
return vs_train_op
# construct the network
def construct_model(self):
self.init_placeholder()
self.loss_cls_reg, offset_pred, loss_reg = self.compute_loss_reg(
self.central_ph_train, self.left_ph_train, self.right_ph_train,
self.offset_ph, self.label_ph)
self.train_op = self.training(self.loss_cls_reg)
eval_op = self.eval(self.central_ph_test, self.left_ph_test,
self.right_ph_test)
return self.loss_cls_reg, self.train_op, eval_op, loss_reg
class ACRN_Model(object):
def __init__(self, batch_size, pool_size, train_csv_path, test_csv_path,
test_visual_feature_dir, train_visual_feature_dir):
self.batch_size = batch_size
self.test_batch_size = 1
self.vs_lr = 0.0001
self.lambda_regression = 0.01
self.alpha = 1.0 / batch_size
#self.alpha=0.06
self.pool_size = pool_size
self.semantic_size = 1024
self.sentence_embedding_size = 4800
self.visual_feature_dim = 4096
self.train_set = TrainingDataSet(train_visual_feature_dir,
train_csv_path, self.batch_size)
self.test_set = TestingDataSet(test_visual_feature_dir, test_csv_path,
self.test_batch_size)
self.context_num = 1
'''
used in training alignment model, CTRL(aln)
'''
def fill_feed_dict_train(self):
image_batch, sentence_batch, offset_batch = self.train_set.next_batch()
input_feed = {
self.visual_featmap_ph_train: image_batch,
self.sentence_ph_train: sentence_batch,
self.offset_ph: offset_batch
}
return input_feed
'''
used in training alignment+regression model, CTRL(reg)
'''
def fill_feed_dict_train_reg(self):
image_batch, sentence_batch, offset_batch = self.train_set.next_batch_iou(
)
input_feed = {
self.visual_featmap_ph_train: image_batch,
self.sentence_ph_train: sentence_batch,
self.offset_ph: offset_batch
}
return input_feed
'''
cross modal processing module
'''
def cross_modal_comb(self, visual_feat, sentence_embed, batch_size):
vv_feature = tf.reshape(tf.tile(visual_feat, [batch_size, 1]),
[batch_size, batch_size, self.semantic_size])
ss_feature = tf.reshape(tf.tile(sentence_embed, [1, batch_size]),
[batch_size, batch_size, self.semantic_size])
vv_feature1 = tf.reshape(vv_feature, [batch_size, batch_size, -1, 1])
ss_feature1 = tf.reshape(ss_feature, [batch_size, batch_size, -1, 1])
pool_vv = tf.nn.avg_pool(vv_feature1,
ksize=[1, 1, self.pool_size, 1],
strides=[1, 1, self.pool_size, 1],
padding='SAME')
pool_ss = tf.nn.avg_pool(ss_feature1,
ksize=[1, 1, self.pool_size, 1],
strides=[1, 1, self.pool_size, 1],
padding='SAME')
shape_vv = pool_vv.get_shape().as_list()
shape_ss = pool_ss.get_shape().as_list()
vv = tf.reshape(pool_vv, [batch_size * batch_size, shape_vv[2], 1])
ss = tf.reshape(
pool_ss,
[batch_size * batch_size, 1, shape_ss[2]]) #batchx batch, fea
print vv.shape, ss.shape
concat_feature = tf.matmul(vv, ss) #batch*batch, 1024*1024
print concat_feature.shape
concat_feature = tf.reshape(concat_feature,
[batch_size, batch_size, -1])
comb_feature = tf.reshape(
tf.concat([vv_feature, ss_feature, concat_feature], 2),
[1, batch_size, batch_size, -1])
return comb_feature
'''
visual semantic inference, including visual semantic alignment and clip location regression
'''
def visual_semantic_infer(self, visual_feature_train, sentence_embed_train,
visual_feature_test, sentence_embed_test):
name = "CTRL_Model"
with tf.variable_scope(name):
print "Building training network...............................\n"
""" embedding into common space dim 1024"""
visual_feature_train = tf.transpose(visual_feature_train,
[0, 2, 1]) # batch num fea
inputs = tf.reshape(visual_feature_train,
[-1, self.visual_feature_dim]) #batch x num,fe
transformed_clip_train = fc(
'v2s_lt', inputs,
output_dim=self.semantic_size) # batch x num, embed
transformed_clip_train = tf.reshape(transformed_clip_train, [
self.batch_size, 2 * self.context_num + 1, self.semantic_size
]) #batch num embe
transformed_sentence_train = fc(
's2s_lt', sentence_embed_train,
output_dim=self.semantic_size) # batch, embed
#### attention part
print "attention part tanh(sum(x_1:t))*tanh(s) "
concat_previous_feature = tf.zeros(
[self.batch_size, 1, self.semantic_size])
for j in range(2 * self.context_num):
now = tf.slice(transformed_clip_train, [0, 0, 0],
[-1, j + 1, -1])
# print now.get_shape().as_list()
now = tf.reduce_sum(now, 1)
# print now.get_shape().as_list()
now = tf.expand_dims(now, 1)
# print now.get_shape().as_list()
concat_previous_feature = tf.concat(
[concat_previous_feature, now], 1) # batch num embed
v = tf.tanh(tf.add(transformed_clip_train,
concat_previous_feature))
relu_t = tf.tanh(transformed_sentence_train) #batch, embed
concat_text = tf.reshape(
tf.tile(relu_t, [1, 2 * self.context_num + 1]), [
self.batch_size, 2 * self.context_num + 1,
self.semantic_size
]) # batch cont_num embed
# computing weight a
e = tf.reduce_sum(tf.multiply(concat_text, v), 2) # batch cont_num
alpha = tf.nn.softmax(e) # batch, num_ctx
a = tf.reshape(tf.tile(alpha, [1, self.semantic_size]), [
self.batch_size, self.semantic_size, 2 * self.context_num + 1
]) # batch 4096 cont_num
visual_feature_train = tf.transpose(transformed_clip_train,
[0, 2, 1]) # batch embed num
input_vision = tf.reduce_sum(tf.multiply(visual_feature_train, a),
2) #batch embed
transformed_clip_train_norm = tf.nn.l2_normalize(input_vision,
dim=1)
transformed_sentence_train_norm = tf.nn.l2_normalize(
transformed_sentence_train, dim=1)
# print transformed_clip_train_norm.shape
# print transformed_sentence_train_norm.shape
# exit()
cross_modal_vec_train = self.cross_modal_comb(
transformed_clip_train_norm, transformed_sentence_train_norm,
self.batch_size) # batch batch 2*conmmon_space_dim
# print cross_modal_vec_train.shape
# exit()
sim_score_mat_train = vs_multilayer.vs_multilayer(
cross_modal_vec_train,
"vs_multilayer_lt",
middle_layer_dim=1000)
sim_score_mat_train = tf.reshape(
sim_score_mat_train, [self.batch_size, self.batch_size, 3])
tf.get_variable_scope().reuse_variables()
print "Building test network...............................\n"
visual_feature_test = tf.transpose(visual_feature_test,
[0, 2, 1]) # batch num fea
inputs = tf.reshape(visual_feature_test,
[-1, self.visual_feature_dim]) #batch x num,fe
transformed_clip_test = fc('v2s_lt',
inputs,
output_dim=self.semantic_size)
transformed_clip_test = tf.reshape(transformed_clip_test, [
self.test_batch_size, 2 * self.context_num + 1,
self.semantic_size
]) #batch num embe
transformed_sentence_test = fc('s2s_lt',
sentence_embed_test,
output_dim=self.semantic_size)
#### attention part
print "attention part tanh(sum(x_1:t))*tanh(s) "
concat_previous_feature = tf.zeros(
[self.test_batch_size, 1, self.semantic_size])
for j in range(2 * self.context_num):
now = tf.slice(transformed_clip_test, [0, 0, 0],
[-1, j + 1, -1])
print now.get_shape().as_list()
now = tf.reduce_sum(now, 1)
print now.get_shape().as_list()
now = tf.expand_dims(now, 1)
print now.get_shape().as_list()
concat_previous_feature = tf.concat(
1, [concat_previous_feature, now]) # batch num embed
v = tf.tanh(tf.add(transformed_clip_test,
concat_previous_feature)) # batchx num, embed
relu_t = tf.tanh(transformed_sentence_test) #batch, feature_embed
concat_text = tf.reshape(
tf.tile(relu_t, [1, 2 * self.context_num + 1]), [
self.test_batch_size, 2 * self.context_num + 1,
self.semantic_size
]) # batch cont_num feature
e = tf.reduce_sum(tf.mul(concat_text, v), 2) # batch cont_num
alpha = tf.nn.softmax(e) # batch, num_ctx
a = tf.reshape(tf.tile(alpha, [1, self.semantic_size]), [
self.test_batch_size, self.semantic_size,
2 * self.context_num + 1
]) # batch embed cont_num
visual_feature_test = tf.transpose(transformed_clip_test,
[0, 2, 1])
input_vision = tf.reduce_sum(tf.mul(visual_feature_test, a),
2) #batch embed
transformed_clip_test_norm = tf.nn.l2_normalize(input_vision,
dim=1)
transformed_sentence_test_norm = tf.nn.l2_normalize(
transformed_sentence_test, dim=1)
cross_modal_vec_test = self.cross_modal_comb(
transformed_clip_test_norm, transformed_sentence_test_norm,
self.test_batch_size)
sim_score_mat_test = vs_multilayer.vs_multilayer(
cross_modal_vec_test,
"vs_multilayer_lt",
reuse=True,
middle_layer_dim=1000)
sim_score_mat_test = tf.reshape(sim_score_mat_test, [3])
return sim_score_mat_train, sim_score_mat_test
'''
compute alignment and regression loss
'''
def compute_loss_reg(self, sim_reg_mat, offset_label):
sim_score_mat, p_reg_mat, l_reg_mat = tf.split(2, 3, sim_reg_mat)
sim_score_mat = tf.reshape(sim_score_mat,
[self.batch_size, self.batch_size])
l_reg_mat = tf.reshape(l_reg_mat, [self.batch_size, self.batch_size])
p_reg_mat = tf.reshape(p_reg_mat, [self.batch_size, self.batch_size])
# unit matrix with -2
I_2 = tf.diag(tf.constant(-2.0, shape=[self.batch_size]))
all1 = tf.constant(1.0, shape=[self.batch_size, self.batch_size])
# | -1 1 1... |
# mask_mat = | 1 -1 -1... |
# | 1 1 -1 ... |
mask_mat = tf.add(I_2, all1)
# loss cls, not considering iou
I = tf.diag(tf.constant(1.0, shape=[self.batch_size]))
I_half = tf.diag(tf.constant(0.5, shape=[self.batch_size]))
batch_para_mat = tf.constant(self.alpha,
shape=[self.batch_size, self.batch_size])
para_mat = tf.add(I, batch_para_mat)
loss_mat = tf.log(tf.add(all1, tf.exp(tf.mul(mask_mat,
sim_score_mat))))
loss_mat = tf.mul(loss_mat, para_mat)
loss_align = tf.reduce_mean(loss_mat)
# regression loss
l_reg_diag = tf.matmul(tf.mul(l_reg_mat, I),
tf.constant(1.0, shape=[self.batch_size, 1]))
p_reg_diag = tf.matmul(tf.mul(p_reg_mat, I),
tf.constant(1.0, shape=[self.batch_size, 1]))
offset_pred = tf.concat(1, (p_reg_diag, l_reg_diag))
loss_reg = tf.reduce_mean(tf.abs(tf.sub(offset_pred, offset_label)))
loss = tf.add(tf.mul(self.lambda_regression, loss_reg), loss_align)
return loss, offset_pred, loss_reg
def init_placeholder(self):
visual_featmap_ph_train = tf.placeholder(
tf.float32,
shape=(
self.batch_size, self.visual_feature_dim,
2 * self.context_num +
1)) # input feature: current clip, pre-contex, and post contex
sentence_ph_train = tf.placeholder(
tf.float32, shape=(self.batch_size, self.sentence_embedding_size))
offset_ph = tf.placeholder(tf.float32, shape=(self.batch_size, 2))
visual_featmap_ph_test = tf.placeholder(
tf.float32,
shape=(
self.test_batch_size, self.visual_feature_dim,
2 * self.context_num +
1)) #input feature: current clip, pre-contex, and post contex
sentence_ph_test = tf.placeholder(tf.float32,
shape=(self.test_batch_size,
self.sentence_embedding_size))
return visual_featmap_ph_train, sentence_ph_train, offset_ph, visual_featmap_ph_test, sentence_ph_test
def get_variables_by_name(self, name_list):
v_list = tf.trainable_variables()
v_dict = {}
for name in name_list:
v_dict[name] = []
for v in v_list:
for name in name_list:
if name in v.name: v_dict[name].append(v)
for name in name_list:
print "Variables of <" + name + ">"
for v in v_dict[name]:
print " " + v.name
return v_dict
def training(self, loss):
v_dict = self.get_variables_by_name(["lt"])
vs_optimizer = tf.train.AdamOptimizer(self.vs_lr, name='vs_adam')
vs_train_op = vs_optimizer.minimize(loss, var_list=v_dict["lt"])
return vs_train_op
def construct_model(self):
# initialize the placeholder
self.visual_featmap_ph_train, self.sentence_ph_train, self.offset_ph, self.visual_featmap_ph_test, self.sentence_ph_test = self.init_placeholder(
)
# build inference network
sim_reg_mat, sim_reg_mat_test = self.visual_semantic_infer(
self.visual_featmap_ph_train, self.sentence_ph_train,
self.visual_featmap_ph_test, self.sentence_ph_test)
# compute loss
self.loss_align_reg, offset_pred, loss_reg = self.compute_loss_reg(
sim_reg_mat, self.offset_ph)
# optimize
self.vs_train_op = self.training(self.loss_align_reg)
return self.loss_align_reg, self.vs_train_op, sim_reg_mat_test, offset_pred, loss_reg
class CBR_Model(object):
""" This is the body of the network we are using
Here you will get access to the network structure, function of training, evaluation
"""
def __init__(self, config):
"""Initialization
"""
self.config = config
self.sess = None
self.saver = None
self.train_clip_path = self.config.train_clip_path
self.background_path = self.config.background_path
self.test_clip_path = self.config.test_clip_path
self.train_flow_feature_dir = self.config.train_flow_feature_dir
self.train_appr_feature_dir = self.config.train_appr_feature_dir
self.test_flow_feature_dir = self.config.test_flow_feature_dir
self.test_appr_feature_dir = self.config.test_appr_feature_dir
self.test_len_dict = self.config.test_len_dict
self.batch_size = self.config.batch_size
self.test_batch_size = 1
self.middle_layer_size = 1000
self.lambda_reg = float(self.config.lambda_reg)
self.action_class_num = self.config.action_class_num
self.feat_type = self.config.feat_type
self.visual_feature_dim = self.config.visual_feature_dim
# Initialize the training data and testing data
self.train_set = TrainingDataSet(self.config,
self.train_flow_feature_dir,
self.train_appr_feature_dir,
self.train_clip_path,
self.background_path)
self.test_set = TestingDataSet(self.config, self.test_flow_feature_dir,
self.test_appr_feature_dir,
self.test_clip_path,
self.test_batch_size,
self.test_len_dict)
# Path to save the summary of the models
self.summary_dir = os.path.join('./summary', self.config.save_name)
if not os.path.exists(self.summary_dir):
os.mkdir(self.summary_dir)
if self.config.issave == 'Yes':
self.model_dir = os.path.join('./model', self.config.save_name)
if not os.path.exists(self.model_dir):
os.mkdir(self.model_dir)
def init_session(self):
"""Create a session in tensorflow
"""
print('Initializing of session')
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.3
) # 30% memory of TITAN is enough
self.sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
self.sess.run(tf.global_variables_initializer())
self.saver = tf.train.Saver(max_to_keep=10)
if self.config.ispretrain == 'Yes':
self.restore_session('model/xx.ckpt')
def get_feed_dict(self, lr_by_step):
"""Prepare training samples in each batch size to the network
"""
image_batch, label_batch, offset_batch, one_hot_label_batch = self.train_set.next_batch(
)
input_feed = {
self.visual_featmap_ph_train: image_batch,
self.label_ph: label_batch,
self.offset_ph: offset_batch,
self.one_hot_label_ph: one_hot_label_batch,
self.vs_lr: lr_by_step
}
return input_feed
def add_loss_op(self,
visual_feature,
offsets,
labels,
one_hot_labels,
name='CBR'):
"""This function is to compute the loss in tensorflow graph
Args:
visual_feature: Tensor, feature, (batch_size, visual_feature_dim)
offsets: Tensor, boundary offset(both to the start and end in frame-level), (batch_size, 2)
labels: Tensor, label, (batch_size)
one_hot_labels: Tensor, one hot label, (batch_size, action_class_num+1)
Returns:
loss: loss_cls + lambda_reg * loss_reg
loss_reg: L1 loss between ground truth offsets and prediction offsets
loss_cls: cross entropy loss
"""
print('Add the standard loss')
cls_reg_vec = vs_multilayer.vs_multilayer(
visual_feature,
name,
middle_layer_dim=self.middle_layer_size,
class_num=self.action_class_num,
dropout=self.config.dropout)
cls_reg_vec = tf.reshape(
cls_reg_vec, [self.batch_size, (self.action_class_num + 1) * 3])
cls_score_vec = cls_reg_vec[:, :self.action_class_num + 1]
start_offset_pred = cls_reg_vec[:, self.action_class_num +
1:(self.action_class_num + 1) * 2]
end_offset_pred = cls_reg_vec[:, (self.action_class_num + 1) * 2:]
# l1 loss
loss_l1 = tf.reduce_mean(tf.abs(cls_score_vec))
# classification loss
loss_cls_vec = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=cls_score_vec, labels=labels)
loss_cls = tf.reduce_mean(loss_cls_vec)
# regression loss
pick_start_offset_pred = []
pick_end_offset_pred = []
for k in range(self.batch_size):
pick_start_offset_pred.append(start_offset_pred[k, labels[k]])
pick_end_offset_pred.append(end_offset_pred[k, labels[k]])
pick_start_offset_pred = tf.reshape(tf.stack(pick_start_offset_pred),
[self.batch_size, 1])
pick_end_offset_pred = tf.reshape(tf.stack(pick_end_offset_pred),
[self.batch_size, 1])
labels_1 = tf.to_float(tf.not_equal(labels, 0))
label_tmp = tf.to_float(tf.reshape(labels_1, [self.batch_size, 1]))
label_for_reg = tf.concat([label_tmp, label_tmp], 1)
offset_pred = tf.concat((pick_start_offset_pred, pick_end_offset_pred),
1)
loss_reg = tf.reduce_mean(
tf.multiply(tf.abs(tf.subtract(offset_pred, offsets)),
label_for_reg))
loss = tf.add(tf.multiply(self.lambda_reg, loss_reg), loss_cls)
if self.config.l1_loss:
loss = tf.add(loss, loss_l1)
else:
loss = loss
tf.summary.scalar("loss", loss)
tf.summary.scalar("loss_reg", loss_reg)
tf.summary.scalar("loss_cls", loss_cls)
return loss, loss_reg, loss_cls
def add_placeholders(self):
"""Add placeholders
"""
print('Add placeholders')
self.visual_featmap_ph_train = tf.placeholder(
tf.float32,
name='train_featmap',
shape=(self.batch_size, self.visual_feature_dim))
self.visual_featmap_ph_test = tf.placeholder(
tf.float32,
name='test_featmap',
shape=(self.test_batch_size, self.visual_feature_dim))
self.label_ph = tf.placeholder(tf.int32,
name='label',
shape=(self.batch_size))
self.offset_ph = tf.placeholder(tf.float32,
name='offset',
shape=(self.batch_size, 2))
self.one_hot_label_ph = tf.placeholder(
tf.float32,
name='one_hot_label',
shape=(self.batch_size, self.action_class_num + 1))
self.vs_lr = tf.placeholder(tf.float32, name='lr')
def add_summary(self):
"""Add summary
"""
print('Add summay')
self.merged = tf.summary.merge_all()
self.file_writer = tf.summary.FileWriter(self.summary_dir,
self.sess.graph)
def save_session(self, step):
"""Save the session if needed
"""
if self.config.issave == 'Yes':
print('Save session')
model_name = os.path.join(self.model_dir, str(step) + '.ckpt')
self.saver.save(self.sess, model_name)
def restore_session(self, dir_model):
"""Restore session
"""
print('Restore the Session')
self.saver.restore(self.sess, dir_model)
def close_session(self):
""" Close session once finished
"""
print('Close session')
self.sess.close()
def predict(self, visual_feature_test):
"""Inference during testing
Args:
visual_feature_test: Tensor, feature, (test_batch_size, visual_feature_dim)
Returns:
sim_score: Tensor, (action_class_num+1)*3 (Note: [0:action_class_num+1]: classification scores;
[action_class_num+1:(action_class_num+1)*2: start offsets; [(action_class_num+1)*2:(action_class_num+1)*3]: end offsets)
"""
print('To predict the label')
sim_score = vs_multilayer.vs_multilayer(
visual_feature_test,
"CBR",
middle_layer_dim=self.middle_layer_size,
class_num=self.action_class_num,
dropout=False,
reuse=True)
sim_score = tf.reshape(sim_score, [(self.action_class_num + 1) * 3])
return sim_score
def get_variables_by_name(self, name_list):
"""Get variables by name
"""
v_list = tf.trainable_variables()
v_dict = {}
for name in name_list:
v_dict[name] = []
for v in v_list:
for name in name_list:
if name in v.name: v_dict[name].append(v)
for name in name_list:
print("Variables of <" + name + ">")
for v in v_dict[name]:
print(" " + v.name)
return v_dict
def add_train_op(self, loss):
"""Add train operation
"""
print('Add train operation')
v_dict = self.get_variables_by_name(["CBR"])
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if self.config.opm_type == 'adam_wd':
vs_optimizer = tf.contrib.opt.extend_with_decoupled_weight_decay(
tf.train.AdamOptimizer)
optimizer = vs_optimizer(weight_decay=1e-4,
learning_rate=self.vs_lr,
name='vs_adam')
with tf.control_dependencies(update_ops):
vs_train_op = optimizer.minimize(loss, var_list=v_dict["CBR"])
elif self.config.opm_type == 'adam':
vs_optimizer = tf.train.AdamOptimizer(self.vs_lr, name='vs_adam')
with tf.control_dependencies(update_ops):
vs_train_op = vs_optimizer.minimize(loss,
var_list=v_dict["CBR"])
return vs_train_op
def build(self):
"""Build the model
"""
print('Construct the network')
self.add_placeholders()
if self.config.norm == 'l2':
visual_featmap_ph_train_norm = tf.nn.l2_normalize(
self.visual_featmap_ph_train, dim=1)
visual_featmap_ph_test_norm = tf.nn.l2_normalize(
self.visual_featmap_ph_test, dim=1)
elif self.config.norm == 'No':
visual_featmap_ph_train_norm = self.visual_featmap_ph_train
visual_featmap_ph_test_norm = self.visual_featmap_ph_test
self.loss, self.loss_reg, self.loss_cls = self.add_loss_op(
visual_featmap_ph_train_norm, self.offset_ph, self.label_ph,
self.one_hot_label_ph)
self.vs_train_op = self.add_train_op(self.loss)
self.vs_eval_op = self.predict(visual_featmap_ph_test_norm)
self.init_session()
def train(self):
"""Training
"""
self.add_summary()
for step in xrange(self.config.max_steps):
# if step <= 3000:
lr = self.config.lr
# else:
# lr = self.config.lr/10
start_time = time.time()
feed_dict = self.get_feed_dict(lr)
duration1 = time.time() - start_time
[_, loss_value, loss_reg_value, loss_cls_value,
summary] = self.sess.run([
self.vs_train_op, self.loss, self.loss_reg, self.loss_cls,
self.merged
],
feed_dict=feed_dict)
duration2 = time.time() - start_time
print(
'Step %d: loss=%.2f, loss_reg=%.2f, loss_cls=%.2f, (%.3f sec),(%.3f sec)'
% (step, loss_value, loss_reg_value, loss_cls_value, duration1,
duration2))
self.file_writer.add_summary(summary, step)
if (step + 1) == 4000 or (step + 1) % self.config.test_steps == 0:
self.save_session(step + 1)
def do_eval_slidingclips(self, save_name):
"""Do evaluation based on proposals and save the coresponding score and offset to a pickle file in './eval/test_results' folder
"""
test_len_dict = tools.load_length_dict(type='test')
reg_result_dict = {}
for k, test_sample in enumerate(self.test_set.test_samples):
reg_result_dict[k] = []
if k % 1000 == 0:
print(str(k) + "/" + str(len(self.test_set.test_samples)))
movie_name = test_sample[0]
init_clip_start = test_sample[1]
init_clip_end = test_sample[2]
clip_start = init_clip_start
clip_end = init_clip_end
final_action_prob = np.zeros([
(self.config.action_class_num + 1) * 3 * self.config.cas_step
])
if clip_start >= clip_end:
reg_result_dict[k].append(final_action_prob)
continue
for i in range(self.config.cas_step):
if clip_start >= clip_end:
break
if self.config.feat_type == 'Pool':
featmap = dataset.get_pooling_feature(
self.test_set.flow_feat_dir,
self.test_set.appr_feat_dir, movie_name, clip_start,
clip_end, self.config.pool_level,
self.config.unit_size, self.config.unit_feature_size,
self.config.fusion_type)
left_feat = dataset.get_left_context_feature(
self.test_set.flow_feat_dir,
self.test_set.appr_feat_dir, movie_name, clip_start,
clip_end, self.config.ctx_num, self.config.unit_size,
self.config.unit_feature_size, self.config.fusion_type)
right_feat = dataset.get_right_context_feature(
self.test_set.flow_feat_dir,
self.test_set.appr_feat_dir, movie_name, clip_start,
clip_end, self.config.ctx_num, self.config.unit_size,
self.config.unit_feature_size, self.config.fusion_type)
mean_ = np.hstack((left_feat, featmap, right_feat))
feat = mean_
elif self.config.feat_type == 'SSN':
feat = dataset.get_SSN_feature(
self.test_set.flow_feat_dir,
self.test_set.appr_feat_dir, movie_name, clip_start,
clip_end, self.config.unit_size,
self.config.unit_feature_size, self.config.fusion_type)
else:
feat = dataset.get_BSP_feature(
self.test_set.flow_feat_dir,
self.test_set.appr_feat_dir, movie_name, clip_start,
clip_end, self.config.unit_size,
self.config.unit_feature_size, self.config.bsp_level)
feat = np.reshape(feat, [1, self.config.visual_feature_dim])
feed_dict = {self.visual_featmap_ph_test: feat}
outputs = self.sess.run(self.vs_eval_op, feed_dict=feed_dict)
action_score = outputs[1:self.config.action_class_num + 1]
action_prob = tools.softmax(action_score)
final_action_prob[(i) * (self.config.action_class_num + 1) *
3:(i + 1) *
(self.config.action_class_num + 1) *
3] = outputs
action_cat = np.argmax(action_prob) + 1
round_reg_end = clip_end + round(outputs[
(self.config.action_class_num + 1) * 2 +
action_cat]) * self.config.unit_size
round_reg_start = clip_start + round(
outputs[self.config.action_class_num + 1 +
action_cat]) * self.config.unit_size
if round_reg_start < 0 or round_reg_end > test_len_dict[
movie_name] - 15 or round_reg_start >= round_reg_end:
round_reg_end = clip_end
round_reg_start = clip_start
reg_end = clip_end + outputs[
(self.config.action_class_num + 1) * 2 +
action_cat] * self.config.unit_size
reg_start = clip_start + outputs[
self.config.action_class_num + 1 +
action_cat] * self.config.unit_size
clip_start = round_reg_start
clip_end = round_reg_end
reg_result_dict[k].append(final_action_prob)
pickle.dump(
reg_result_dict,
open("./eval/test_results/" + save_name + "_outputs.pkl", "wb"))
class CTRL_Model(object):
def __init__(self, batch_size, train_csv_path, test_csv_path,
test_visual_feature_dir, train_visual_feature_dir):
self.batch_size = batch_size
self.test_batch_size = 1
self.vs_lr = 0.005
self.lambda_regression = 0.01
self.alpha = 1.0 / batch_size
self.semantic_size = 1024 # the size of visual and semantic comparison size
self.sentence_embedding_size = 4800
self.visual_feature_dim = 4096 * 3
self.train_set = TrainingDataSet(train_visual_feature_dir,
train_csv_path, self.batch_size)
self.test_set = TestingDataSet(test_visual_feature_dir, test_csv_path,
self.test_batch_size)
'''
used in training alignment model, CTRL(aln)
'''
def fill_feed_dict_train(self):
image_batch, sentence_batch, offset_batch = self.train_set.next_batch()
input_feed = {
self.visual_featmap_ph_train: image_batch,
self.sentence_ph_train: sentence_batch,
self.offset_ph: offset_batch
}
return input_feed
'''
used in training alignment+regression model, CTRL(reg)
'''
def fill_feed_dict_train_reg(self):
image_batch, sentence_batch, offset_batch = self.train_set.next_batch_iou(
)
input_feed = {
self.visual_featmap_ph_train: image_batch,
self.sentence_ph_train: sentence_batch,
self.offset_ph: offset_batch
}
return input_feed
'''
cross modal processing module
'''
def cross_modal_comb(self, visual_feat, sentence_embed, batch_size):
vv_feature = tf.reshape(tf.tile(visual_feat, [batch_size, 1]),
[batch_size, batch_size, self.semantic_size])
ss_feature = tf.reshape(tf.tile(sentence_embed, [1, batch_size]),
[batch_size, batch_size, self.semantic_size])
concat_feature = tf.reshape(
tf.concat([vv_feature, ss_feature], 2),
[batch_size, batch_size, self.semantic_size + self.semantic_size])
print concat_feature.get_shape().as_list()
mul_feature = tf.multiply(vv_feature, ss_feature)
add_feature = tf.add(vv_feature, ss_feature)
comb_feature = tf.reshape(
tf.concat([mul_feature, add_feature, concat_feature], 2),
[1, batch_size, batch_size, self.semantic_size * 4])
return comb_feature
'''
visual semantic inference, including visual semantic alignment and clip location regression
'''
def visual_semantic_infer(self, visual_feature_train, sentence_embed_train,
visual_feature_test, sentence_embed_test):
name = "CTRL_Model"
with tf.variable_scope(name):
print "Building training network...............................\n"
transformed_clip_train = fc('v2s_lt',
visual_feature_train,
output_dim=self.semantic_size)
transformed_clip_train_norm = tf.nn.l2_normalize(
transformed_clip_train, dim=1)
transformed_sentence_train = fc('s2s_lt',
sentence_embed_train,
output_dim=self.semantic_size)
transformed_sentence_train_norm = tf.nn.l2_normalize(
transformed_sentence_train, dim=1)
cross_modal_vec_train = self.cross_modal_comb(
transformed_clip_train_norm, transformed_sentence_train_norm,
self.batch_size)
sim_score_mat_train = vs_multilayer.vs_multilayer(
cross_modal_vec_train,
"vs_multilayer_lt",
middle_layer_dim=1000)
sim_score_mat_train = tf.reshape(
sim_score_mat_train, [self.batch_size, self.batch_size, 3])
tf.get_variable_scope().reuse_variables()
print "Building test network...............................\n"
transformed_clip_test = fc('v2s_lt',
visual_feature_test,
output_dim=self.semantic_size)
transformed_clip_test_norm = tf.nn.l2_normalize(
transformed_clip_test, dim=1)
transformed_sentence_test = fc('s2s_lt',
sentence_embed_test,
output_dim=self.semantic_size)
transformed_sentence_test_norm = tf.nn.l2_normalize(
transformed_sentence_test, dim=1)
cross_modal_vec_test = self.cross_modal_comb(
transformed_clip_test_norm, transformed_sentence_test_norm,
self.test_batch_size)
sim_score_mat_test = vs_multilayer.vs_multilayer(
cross_modal_vec_test,
"vs_multilayer_lt",
reuse=True,
middle_layer_dim=1000)
sim_score_mat_test = tf.reshape(sim_score_mat_test, [3])
return sim_score_mat_train, sim_score_mat_test
'''
compute alignment and regression loss
'''
def compute_loss_reg(self, sim_reg_mat, offset_label):
sim_score_mat, p_reg_mat, l_reg_mat = tf.split(sim_reg_mat, 3, 2)
sim_score_mat = tf.reshape(sim_score_mat,
[self.batch_size, self.batch_size])
l_reg_mat = tf.reshape(l_reg_mat, [self.batch_size, self.batch_size])
p_reg_mat = tf.reshape(p_reg_mat, [self.batch_size, self.batch_size])
# unit matrix with -2
I_2 = tf.diag(tf.constant(-2.0, shape=[self.batch_size]))
all1 = tf.constant(1.0, shape=[self.batch_size, self.batch_size])
# | -1 1 1... |
# mask_mat = | 1 -1 -1... |
# | 1 1 -1 ... |
mask_mat = tf.add(I_2, all1)
# loss cls, not considering iou
I = tf.diag(tf.constant(1.0, shape=[self.batch_size]))
I_half = tf.diag(tf.constant(0.5, shape=[self.batch_size]))
batch_para_mat = tf.constant(self.alpha,
shape=[self.batch_size, self.batch_size])
para_mat = tf.add(I, batch_para_mat)
loss_mat = tf.log(
tf.add(all1, tf.exp(tf.multiply(mask_mat, sim_score_mat))))
loss_mat = tf.multiply(loss_mat, para_mat)
loss_align = tf.reduce_mean(loss_mat)
# regression loss
l_reg_diag = tf.matmul(tf.multiply(l_reg_mat, I),
tf.constant(1.0, shape=[self.batch_size, 1]))
p_reg_diag = tf.matmul(tf.multiply(p_reg_mat, I),
tf.constant(1.0, shape=[self.batch_size, 1]))
offset_pred = tf.concat((p_reg_diag, l_reg_diag), 1)
loss_reg = tf.reduce_mean(
tf.abs(tf.subtract(offset_pred, offset_label)))
loss = tf.add(tf.multiply(self.lambda_regression, loss_reg),
loss_align)
return loss, offset_pred, loss_reg
def init_placeholder(self):
visual_featmap_ph_train = tf.placeholder(
tf.float32, shape=(self.batch_size, self.visual_feature_dim))
sentence_ph_train = tf.placeholder(
tf.float32, shape=(self.batch_size, self.sentence_embedding_size))
offset_ph = tf.placeholder(tf.float32, shape=(self.batch_size, 2))
visual_featmap_ph_test = tf.placeholder(
tf.float32, shape=(self.test_batch_size, self.visual_feature_dim))
sentence_ph_test = tf.placeholder(tf.float32,
shape=(self.test_batch_size,
self.sentence_embedding_size))
return visual_featmap_ph_train, sentence_ph_train, offset_ph, visual_featmap_ph_test, sentence_ph_test
def get_variables_by_name(self, name_list):
v_list = tf.trainable_variables()
v_dict = {}
for name in name_list:
v_dict[name] = []
for v in v_list:
for name in name_list:
if name in v.name: v_dict[name].append(v)
for name in name_list:
print "Variables of <" + name + ">"
for v in v_dict[name]:
print " " + v.name
return v_dict
def training(self, loss):
v_dict = self.get_variables_by_name(["lt"])
vs_optimizer = tf.train.AdamOptimizer(self.vs_lr, name='vs_adam')
vs_train_op = vs_optimizer.minimize(loss, var_list=v_dict["lt"])
return vs_train_op
def construct_model(self):
# initialize the placeholder
self.visual_featmap_ph_train, self.sentence_ph_train, self.offset_ph, self.visual_featmap_ph_test, self.sentence_ph_test = self.init_placeholder(
)
# build inference network
sim_reg_mat, sim_reg_mat_test = self.visual_semantic_infer(
self.visual_featmap_ph_train, self.sentence_ph_train,
self.visual_featmap_ph_test, self.sentence_ph_test)
# compute loss
self.loss_align_reg, offset_pred, loss_reg = self.compute_loss_reg(
sim_reg_mat, self.offset_ph)
# optimize
self.vs_train_op = self.training(self.loss_align_reg)
return self.loss_align_reg, self.vs_train_op, sim_reg_mat_test, offset_pred, loss_reg
class CBR_Model(object):
def __init__(self, batch_size, ctx_num, unit_size, unit_feature_size,
action_class_num, lr, lambda_reg, train_clip_path,
background_path, test_clip_path, train_flow_feature_dir,
train_appr_feature_dir, test_flow_feature_dir,
test_appr_feature_dir):
self.batch_size = batch_size
self.test_batch_size = 1
self.middle_layer_size = 1000
self.vs_lr = lr
self.lambda_reg = lambda_reg
self.action_class_num = action_class_num
self.visual_feature_dim = unit_feature_size * 3
self.train_set = TrainingDataSet(train_flow_feature_dir,
train_appr_feature_dir,
train_clip_path, background_path,
batch_size, ctx_num, unit_size,
unit_feature_size, action_class_num)
self.test_set = TestingDataSet(test_flow_feature_dir,
test_appr_feature_dir, test_clip_path,
self.test_batch_size, unit_size)
def fill_feed_dict_train(self):
image_batch, label_batch, offset_batch, one_hot_label_batch = self.train_set.next_batch(
)
input_feed = {
self.visual_featmap_ph_train: image_batch,
self.label_ph: label_batch,
self.offset_ph: offset_batch,
self.one_hot_label_ph: one_hot_label_batch
}
return input_feed
def compute_loss_reg(self, visual_feature, offsets, labels,
one_hot_labels):
cls_reg_vec = vs_multilayer.vs_multilayer(
visual_feature,
"CBR",
middle_layer_dim=self.middle_layer_size,
output_layer_dim=(self.action_class_num + 1) * 3)
cls_reg_vec = tf.reshape(
cls_reg_vec, [self.batch_size, (self.action_class_num + 1) * 3])
cls_score_vec = cls_reg_vec[:, :self.action_class_num + 1]
start_offset_pred = cls_reg_vec[:, self.action_class_num +
1:(self.action_class_num + 1) * 2]
end_offset_pred = cls_reg_vec[:, (self.action_class_num + 1) * 2:]
#classification loss
loss_cls_vec = tf.nn.sparse_softmax_cross_entropy_with_logits(
cls_score_vec, labels)
loss_cls = tf.reduce_mean(loss_cls_vec)
# regression loss
pick_start_offset_pred = []
pick_end_offset_pred = []
for k in range(self.batch_size):
pick_start_offset_pred.append(start_offset_pred[k, labels[k]])
pick_end_offset_pred.append(end_offset_pred[k, labels[k]])
pick_start_offset_pred = tf.reshape(tf.stack(pick_start_offset_pred),
[self.batch_size, 1])
pick_end_offset_pred = tf.reshape(tf.stack(pick_end_offset_pred),
[self.batch_size, 1])
labels_1 = tf.to_float(tf.not_equal(labels, 0))
label_tmp = tf.to_float(tf.reshape(labels_1, [self.batch_size, 1]))
label_for_reg = tf.concat(1, [label_tmp, label_tmp])
offset_pred = tf.concat(1,
(pick_start_offset_pred, pick_end_offset_pred))
loss_reg = tf.reduce_mean(
tf.mul(tf.abs(tf.sub(offset_pred, offsets)), label_for_reg))
loss = tf.add(tf.mul(self.lambda_reg, loss_reg), loss_cls)
return loss, loss_reg
def init_placeholder(self):
visual_featmap_ph_train = tf.placeholder(
tf.float32, shape=(self.batch_size, self.visual_feature_dim))
label_ph = tf.placeholder(tf.int32, shape=(self.batch_size))
offset_ph = tf.placeholder(tf.float32, shape=(self.batch_size, 2))
one_hot_label_ph = tf.placeholder(tf.float32,
shape=(self.batch_size,
self.action_class_num + 1))
visual_featmap_ph_test = tf.placeholder(
tf.float32, shape=(self.test_batch_size, self.visual_feature_dim))
return visual_featmap_ph_train, visual_featmap_ph_test, label_ph, offset_ph, one_hot_label_ph
def eval(self, visual_feature_test):
sim_score = vs_multilayer.vs_multilayer(
visual_feature_test,
"CBR",
middle_layer_dim=self.middle_layer_size,
output_layer_dim=(self.action_class_num + 1) * 3,
dropout=False,
reuse=True)
sim_score = tf.reshape(sim_score, [(self.action_class_num + 1) * 3])
return sim_score
def get_variables_by_name(self, name_list):
v_list = tf.trainable_variables()
v_dict = {}
for name in name_list:
v_dict[name] = []
for v in v_list:
for name in name_list:
if name in v.name: v_dict[name].append(v)
for name in name_list:
print "Variables of <" + name + ">"
for v in v_dict[name]:
print " " + v.name
return v_dict
def training(self, loss):
v_dict = self.get_variables_by_name(["CBR"])
vs_optimizer = tf.train.AdamOptimizer(self.vs_lr, name='vs_adam')
vs_train_op = vs_optimizer.minimize(loss, var_list=v_dict["CBR"])
return vs_train_op
def construct_model(self):
#construct the network:
self.visual_featmap_ph_train, self.visual_featmap_ph_test, self.label_ph, self.offset_ph, self.one_hot_label_ph = self.init_placeholder(
)
visual_featmap_ph_train_norm = tf.nn.l2_normalize(
self.visual_featmap_ph_train, dim=1)
visual_featmap_ph_test_norm = tf.nn.l2_normalize(
self.visual_featmap_ph_test, dim=1)
self.loss, loss_reg = self.compute_loss_reg(
visual_featmap_ph_train_norm, self.offset_ph, self.label_ph,
self.one_hot_label_ph)
self.vs_train_op = self.training(self.loss)
vs_eval_op = self.eval(visual_featmap_ph_test_norm)
return self.loss, self.vs_train_op, vs_eval_op, loss_reg