def predict(self, other_data, _intv=None):
'''
Returns a RDD object which stores index and predictions
@param
other_data should also be a RDD object;
but it does NOT have class label for each sample
each item is stored as (index, features)
length is the length of other_data
'''
from utils import cdist,vote
# use intv when splitting one test into multiple chunks
if _intv == None:
length = other_data.count()
_intv = range(length)
# Create pair: each test point is associated with a subgroup of train data
pairs = other_data.cartesian(self.data_feature)
predictions = []
for test_idx in _intv:
dist_label_tuple_list = []
# get subset of this test index; collect to do for loop
idx_pairs = pairs.filter(lambda (testpoint, trainsubgroup): testpoint[0] == test_idx).collect()
# loop through each pair
for idx_p in idx_pairs:
# find out the train subgroup
train_subgroup_idx = idx_p[1][0]
# Their Class
C = self.data_label.filter(lambda (ind, subgroup): ind == train_subgroup_idx).collect()[0]
dist_label_tuple_list.extend(cdist(idx_p[0], idx_p[1], C, self.k))
predictions.append((test_idx, vote(dist_label_tuple_list, self.k)))
del idx_pairs
return predictions
def predict(self, other_data, _intv=None):
'''
Returns a RDD object which stores index and predictions
@param
other_data should also be a RDD object;
but it does NOT have class label for each sample
each item is stored as (index, features)
length is the length of other_data
'''
from utils import cdist, vote
# use intv when splitting one test into multiple chunks
if _intv == None:
length = other_data.count()
_intv = range(length)
# Create pair: each test point is associated with a subgroup of train data
pairs = other_data.cartesian(self.data_feature)
predictions = []
for test_idx in _intv:
dist_label_tuple_list = []
# get subset of this test index; collect to do for loop
idx_pairs = pairs.filter(lambda (testpoint, trainsubgroup):
testpoint[0] == test_idx).collect()
# loop through each pair
for idx_p in idx_pairs:
# find out the train subgroup
train_subgroup_idx = idx_p[1][0]
# Their Class
C = self.data_label.filter(lambda (ind, subgroup): ind ==
train_subgroup_idx).collect()[0]
dist_label_tuple_list.extend(
cdist(idx_p[0], idx_p[1], C, self.k))
predictions.append((test_idx, vote(dist_label_tuple_list, self.k)))
del idx_pairs
return predictions
def main():
cfg = TrainConfig().parse()
print(cfg.name)
result_dir = os.path.join(
cfg.result_root,
cfg.name + '_' + datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S'))
if not os.path.isdir(result_dir):
os.makedirs(result_dir)
utils.write_configure_to_file(cfg, result_dir)
np.random.seed(seed=cfg.seed)
# prepare dataset
train_session = cfg.train_session
train_set = prepare_multimodal_dataset(cfg.feature_root, train_session,
cfg.feat, cfg.label_root)
if cfg.task == "supervised": # fully supervised task
train_set = train_set[:cfg.label_num]
batch_per_epoch = len(train_set) // cfg.sess_per_batch
labeled_session = train_session[:cfg.label_num]
val_session = cfg.val_session
val_set = prepare_multimodal_dataset(cfg.feature_root, val_session,
cfg.feat, cfg.label_root)
# construct the graph
with tf.Graph().as_default():
tf.set_random_seed(cfg.seed)
global_step = tf.Variable(0, trainable=False)
lr_ph = tf.placeholder(tf.float32, name='learning_rate')
####################### Load models here ########################
sensors_emb_dim = 32
segment_emb_dim = 32
with tf.variable_scope("modality_core"):
# load backbone model
if cfg.network == "convtsn":
model_emb = networks.ConvTSN(n_seg=cfg.num_seg,
emb_dim=cfg.emb_dim)
elif cfg.network == "convrtsn":
model_emb = networks.ConvRTSN(n_seg=cfg.num_seg,
emb_dim=cfg.emb_dim)
elif cfg.network == "convbirtsn":
model_emb = networks.ConvBiRTSN(n_seg=cfg.num_seg,
emb_dim=cfg.emb_dim)
else:
raise NotImplementedError
input_ph = tf.placeholder(
tf.float32, shape=[None, cfg.num_seg, None, None, None])
dropout_ph = tf.placeholder(tf.float32, shape=[])
model_emb.forward(input_ph,
dropout_ph) # for lstm has variable scope
with tf.variable_scope("sensors"):
model_output_sensors = networks.OutputLayer(
n_input=cfg.emb_dim, n_output=sensors_emb_dim)
with tf.variable_scope("segment"):
model_output_segment = networks.OutputLayer(
n_input=cfg.emb_dim, n_output=segment_emb_dim)
lambda_mul_ph = tf.placeholder(tf.float32, shape=[])
with tf.variable_scope("modality_sensors"):
model_emb_sensors = networks.RTSN(n_seg=cfg.num_seg,
emb_dim=sensors_emb_dim)
input_sensors_ph = tf.placeholder(tf.float32,
shape=[None, cfg.num_seg, 8])
model_emb_sensors.forward(input_sensors_ph, dropout_ph)
var_list = {}
for v in tf.global_variables():
if v.op.name.startswith("modality_sensors"):
var_list[v.op.name.replace("modality_sensors/", "")] = v
restore_saver_sensors = tf.train.Saver(var_list)
with tf.variable_scope("modality_segment"):
model_emb_segment = networks.RTSN(n_seg=cfg.num_seg,
emb_dim=segment_emb_dim,
n_input=357)
input_segment_ph = tf.placeholder(tf.float32,
shape=[None, cfg.num_seg, 357])
model_emb_segment.forward(input_segment_ph, dropout_ph)
var_list = {}
for v in tf.global_variables():
if v.op.name.startswith("modality_segment"):
var_list[v.op.name.replace("modality_segment/", "")] = v
restore_saver_segment = tf.train.Saver(var_list)
############################# Forward Pass #############################
if cfg.normalized:
embedding = tf.nn.l2_normalize(model_emb.hidden,
axis=-1,
epsilon=1e-10)
embedding_sensors = tf.nn.l2_normalize(model_emb_sensors.hidden,
axis=-1,
epsilon=1e-10)
embedding_segment = tf.nn.l2_normalize(model_emb_segment.hidden,
axis=-1,
epsilon=1e-10)
else:
embedding = model_emb.hidden
embedding_sensors = model_emb_sensors.hidden
embedding_segment = model_emb_segment.hidden
# get the number of unsupervised training
unsup_num = tf.shape(input_sensors_ph)[0]
# variable for visualizing the embeddings
emb_var = tf.Variable(tf.zeros([1116, cfg.emb_dim], dtype=tf.float32),
name='embeddings')
set_emb = tf.assign(emb_var, embedding, validate_shape=False)
# calculated for monitoring all-pair embedding distance
diffs = utils.all_diffs_tf(embedding, embedding)
all_dist = utils.cdist_tf(diffs)
tf.summary.histogram('embedding_dists', all_dist)
# split embedding into anchor, positive and negative and calculate triplet loss
anchor, positive, negative = tf.unstack(
tf.reshape(embedding[:-unsup_num], [-1, 3, cfg.emb_dim]), 3, 1)
metric_loss = networks.triplet_loss(anchor, positive, negative,
cfg.alpha)
model_output_sensors.forward(tf.nn.relu(embedding[-unsup_num:]),
dropout_ph)
logits_sensors = model_output_sensors.logits
model_output_segment.forward(tf.nn.relu(embedding[-unsup_num:]),
dropout_ph)
logits_segment = model_output_segment.logits
# MSE loss
MSE_loss_sensors = tf.losses.mean_squared_error(
embedding_sensors, logits_sensors) / sensors_emb_dim
MSE_loss_segment = tf.losses.mean_squared_error(
embedding_sensors, logits_segment) / segment_emb_dim
MSE_loss = MSE_loss_sensors + MSE_loss_segment
regularization_loss = tf.reduce_sum(
tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
total_loss = tf.cond(
tf.equal(unsup_num,
tf.shape(embedding)[0]), lambda: MSE_loss * lambda_mul_ph
+ regularization_loss * cfg.lambda_l2, lambda: metric_loss +
MSE_loss * lambda_mul_ph + regularization_loss * cfg.lambda_l2)
tf.summary.scalar('learning_rate', lr_ph)
# only train the core branch
train_var_list = [
v for v in tf.global_variables()
if v.op.name.startswith("modality_core")
]
train_op = utils.optimize(total_loss, global_step, cfg.optimizer,
lr_ph, train_var_list)
saver = tf.train.Saver(max_to_keep=10)
summary_op = tf.summary.merge_all()
#########################################################################
# session iterator for session sampling
feat_paths_ph = tf.placeholder(tf.string,
shape=[None, cfg.sess_per_batch])
feat2_paths_ph = tf.placeholder(tf.string,
shape=[None, cfg.sess_per_batch])
feat3_paths_ph = tf.placeholder(tf.string,
shape=[None, cfg.sess_per_batch])
label_paths_ph = tf.placeholder(tf.string,
shape=[None, cfg.sess_per_batch])
train_data = multimodal_session_generator(
feat_paths_ph,
feat2_paths_ph,
feat3_paths_ph,
label_paths_ph,
sess_per_batch=cfg.sess_per_batch,
num_threads=2,
shuffled=False,
preprocess_func=[
model_emb.prepare_input, model_emb_sensors.prepare_input,
model_emb_segment.prepare_input
])
train_sess_iterator = train_data.make_initializable_iterator()
next_train = train_sess_iterator.get_next()
# prepare validation data
val_sess = []
val_feats = []
val_feats2 = []
val_feats3 = []
val_labels = []
val_boundaries = []
for session in val_set:
session_id = os.path.basename(session[1]).split('_')[0]
eve_batch, lab_batch, boundary = load_data_and_label(
session[0], session[-1], model_emb.prepare_input_test
) # use prepare_input_test for testing time
val_feats.append(eve_batch)
val_labels.append(lab_batch)
val_sess.extend([session_id] * eve_batch.shape[0])
val_boundaries.extend(boundary)
eve2_batch, _, _ = load_data_and_label(
session[1], session[-1], model_emb_sensors.prepare_input_test)
val_feats2.append(eve2_batch)
eve3_batch, _, _ = load_data_and_label(
session[2], session[-1], model_emb_segment.prepare_input_test)
val_feats3.append(eve3_batch)
val_feats = np.concatenate(val_feats, axis=0)
val_feats2 = np.concatenate(val_feats2, axis=0)
val_feats3 = np.concatenate(val_feats3, axis=0)
val_labels = np.concatenate(val_labels, axis=0)
print("Shape of val_feats: ", val_feats.shape)
# generate metadata.tsv for visualize embedding
with open(os.path.join(result_dir, 'metadata_val.tsv'), 'w') as fout:
fout.write('id\tlabel\tsession_id\tstart\tend\n')
for i in range(len(val_sess)):
fout.write('{0}\t{1}\t{2}\t{3}\t{4}\n'.format(
i, val_labels[i, 0], val_sess[i], val_boundaries[i][0],
val_boundaries[i][1]))
#########################################################################
# Start running the graph
if cfg.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = cfg.gpu
gpu_options = tf.GPUOptions(allow_growth=True)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
summary_writer = tf.summary.FileWriter(result_dir, sess.graph)
with sess.as_default():
sess.run(tf.global_variables_initializer())
print("Restoring sensors model: %s" % cfg.sensors_path)
restore_saver_sensors.restore(sess, cfg.sensors_path)
print("Restoring segment model: %s" % cfg.segment_path)
restore_saver_segment.restore(sess, cfg.segment_path)
# load pretrain model, if needed
if cfg.model_path:
print("Restoring pretrained model: %s" % cfg.model_path)
saver.restore(sess, cfg.model_path)
################## Training loop ##################
epoch = -1
while epoch < cfg.max_epochs - 1:
step = sess.run(global_step, feed_dict=None)
epoch = step // batch_per_epoch
# learning rate schedule, reference: "In defense of Triplet Loss"
if epoch < cfg.static_epochs:
learning_rate = cfg.learning_rate
else:
learning_rate = cfg.learning_rate * \
0.01**((epoch-cfg.static_epochs)/(cfg.max_epochs-cfg.static_epochs))
# prepare data for this epoch
random.shuffle(train_set)
paths = list(zip(*[iter(train_set)] * cfg.sess_per_batch))
feat_paths = [[p[0] for p in path] for path in paths]
feat2_paths = [[p[1] for p in path] for path in paths]
feat3_paths = [[p[2] for p in path] for path in paths]
label_paths = [[p[-1] for p in path] for path in paths]
sess.run(train_sess_iterator.initializer,
feed_dict={
feat_paths_ph: feat_paths,
feat2_paths_ph: feat2_paths,
feat3_paths_ph: feat3_paths,
label_paths_ph: label_paths
})
# for each epoch
batch_count = 1
while True:
try:
##################### Data loading ########################
start_time = time.time()
eve, eve_sensors, eve_segment, lab, batch_sess = sess.run(
next_train)
# for memory concern, 1000 events are used in maximum
if eve.shape[0] > 1000:
idx = np.random.permutation(eve.shape[0])[:1000]
eve = eve[idx]
eve_sensors = eve_sensors[idx]
eve_segment = eve_segment[idx]
lab = lab[idx]
batch_sess = batch_sess[idx]
load_time = time.time() - start_time
##################### Triplet selection #####################
start_time = time.time()
# for labeled sessions, use facenet sampling
eve_labeled = []
lab_labeled = []
for i in range(eve.shape[0]):
# FIXME: use decode again to get session_id str
if batch_sess[i, 0].decode() in labeled_session:
eve_labeled.append(eve[i])
lab_labeled.append(lab[i])
if len(eve_labeled): # if labeled sessions exist
eve_labeled = np.stack(eve_labeled, axis=0)
lab_labeled = np.stack(lab_labeled, axis=0)
# Get the embeddings of all events
eve_embedding = np.zeros(
(eve_labeled.shape[0], cfg.emb_dim),
dtype='float32')
for start, end in zip(
range(0, eve_labeled.shape[0],
cfg.batch_size),
range(
cfg.batch_size,
eve_labeled.shape[0] + cfg.batch_size,
cfg.batch_size)):
end = min(end, eve_labeled.shape[0])
emb = sess.run(embedding,
feed_dict={
input_ph:
eve_labeled[start:end],
dropout_ph: 1.0
})
eve_embedding[start:end] = np.copy(emb)
# Second, sample triplets within sampled sessions
all_diff = utils.all_diffs(eve_embedding,
eve_embedding)
triplet_input_idx, active_count = utils.select_triplets_facenet(
lab_labeled,
utils.cdist(all_diff, metric=cfg.metric),
cfg.triplet_per_batch,
cfg.alpha,
num_negative=cfg.num_negative)
if len(triplet_input_idx) == 0:
triplet_input = eve_labeled[triplet_input_idx]
else:
active_count = -1
# for all sessions in the batch
perm_idx = np.random.permutation(eve.shape[0])
perm_idx = perm_idx[:min(3 * (len(perm_idx) // 3), 3 *
cfg.triplet_per_batch)]
mul_input = eve[perm_idx]
if len(eve_labeled) and triplet_input_idx is not None:
triplet_input = np.concatenate(
(triplet_input, mul_input), axis=0)
else:
triplet_input = mul_input
sensors_input = eve_sensors[perm_idx]
segment_input = eve_segment[perm_idx]
##################### Start training ########################
# supervised initialization
if epoch < cfg.multimodal_epochs:
if not len(eve_labeled
): # if no labeled sessions exist
continue
err, mse_err, _, step, summ = sess.run(
[
total_loss, MSE_loss, train_op,
global_step, summary_op
],
feed_dict={
input_ph: triplet_input,
input_sensors_ph: sensors_input,
dropout_ph: cfg.keep_prob,
lambda_mul_ph: 0.0,
lr_ph: learning_rate
})
else:
print(triplet_input.shape)
err, mse_err1, mse_err2, _, step, summ = sess.run(
[
total_loss, MSE_loss_sensors,
MSE_loss_segment, train_op, global_step,
summary_op
],
feed_dict={
input_ph: triplet_input,
input_sensors_ph: sensors_input,
input_segment_ph: segment_input,
dropout_ph: cfg.keep_prob,
lambda_mul_ph: cfg.lambda_multimodal,
lr_ph: learning_rate
})
train_time = time.time() - start_time
print ("%s\tEpoch: [%d][%d/%d]\tEvent num: %d\tLoad time: %.3f\tTrain_time: %.3f\tLoss %.4f" % \
(cfg.name, epoch+1, batch_count, batch_per_epoch, eve.shape[0], load_time, train_time, err))
summary = tf.Summary(value=[
tf.Summary.Value(tag="train_loss",
simple_value=err),
tf.Summary.Value(tag="active_count",
simple_value=active_count),
tf.Summary.Value(
tag="triplet_num",
simple_value=(triplet_input.shape[0] -
sensors_input.shape[0]) // 3),
tf.Summary.Value(tag="MSE_loss_sensors",
simple_value=mse_err1),
tf.Summary.Value(tag="MSE_loss_segment",
simple_value=mse_err2)
])
summary_writer.add_summary(summary, step)
summary_writer.add_summary(summ, step)
batch_count += 1
except tf.errors.OutOfRangeError:
print("Epoch %d done!" % (epoch + 1))
break
# validation on val_set
print("Evaluating on validation set...")
val_err1, val_err2, val_embeddings, _ = sess.run(
[MSE_loss_sensors, MSE_loss_segment, embedding, set_emb],
feed_dict={
input_ph: val_feats,
input_sensors_ph: val_feats2,
input_segment_ph: val_feats3,
dropout_ph: 1.0
})
mAP, mPrec = utils.evaluate_simple(val_embeddings, val_labels)
summary = tf.Summary(value=[
tf.Summary.Value(tag="Valiation mAP", simple_value=mAP),
tf.Summary.Value(tag="Validation [email protected]",
simple_value=mPrec),
tf.Summary.Value(tag="Validation mse loss sensors",
simple_value=val_err1),
tf.Summary.Value(tag="Validation mse loss segment",
simple_value=val_err2)
])
summary_writer.add_summary(summary, step)
print("Epoch: [%d]\tmAP: %.4f\tmPrec: %.4f" %
(epoch + 1, mAP, mPrec))
# config for embedding visualization
config = projector.ProjectorConfig()
visual_embedding = config.embeddings.add()
visual_embedding.tensor_name = emb_var.name
visual_embedding.metadata_path = os.path.join(
result_dir, 'metadata_val.tsv')
projector.visualize_embeddings(summary_writer, config)
# save model
saver.save(sess,
os.path.join(result_dir, cfg.name + '.ckpt'),
global_step=step)
def main():
cfg = TrainConfig().parse()
print(cfg.name)
result_dir = os.path.join(
cfg.result_root,
cfg.name + '_' + datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S'))
if not os.path.isdir(result_dir):
os.makedirs(result_dir)
utils.write_configure_to_file(cfg, result_dir)
np.random.seed(seed=cfg.seed)
# prepare dataset
train_session = cfg.train_session
train_set = prepare_multimodal_dataset(cfg.feature_root, train_session,
cfg.feat, cfg.label_root)
if cfg.task == "supervised": # fully supervised task
train_set = train_set[:cfg.label_num]
batch_per_epoch = len(train_set) // cfg.sess_per_batch
labeled_session = train_session[:cfg.label_num]
val_session = cfg.val_session
val_set = prepare_multimodal_dataset(cfg.feature_root, val_session,
cfg.feat, cfg.label_root)
# construct the graph
with tf.Graph().as_default():
tf.set_random_seed(cfg.seed)
global_step = tf.Variable(0, trainable=False)
lr_ph = tf.placeholder(tf.float32, name='learning_rate')
####################### Load models here ########################
sensors_emb_dim = 32
segment_emb_dim = 32
with tf.variable_scope("modality_core"):
# load backbone model
if cfg.network == "convtsn":
model_emb = networks.ConvTSN(n_seg=cfg.num_seg,
emb_dim=cfg.emb_dim)
elif cfg.network == "convrtsn":
model_emb = networks.ConvRTSN(n_seg=cfg.num_seg,
emb_dim=cfg.emb_dim)
elif cfg.network == "convbirtsn":
model_emb = networks.ConvBiRTSN(n_seg=cfg.num_seg,
emb_dim=cfg.emb_dim)
else:
raise NotImplementedError
input_ph = tf.placeholder(
tf.float32, shape=[None, cfg.num_seg, None, None, None])
dropout_ph = tf.placeholder(tf.float32, shape=[])
model_emb.forward(input_ph,
dropout_ph) # for lstm has variable scope
with tf.variable_scope("modality_sensors"):
model_emb_sensors = networks.RTSN(n_seg=cfg.num_seg,
emb_dim=sensors_emb_dim)
model_pairsim_sensors = networks.PDDM(n_input=sensors_emb_dim)
input_sensors_ph = tf.placeholder(tf.float32,
shape=[None, cfg.num_seg, 8])
model_emb_sensors.forward(input_sensors_ph, dropout_ph)
var_list = {}
for v in tf.global_variables():
if v.op.name.startswith("modality_sensors"):
var_list[v.op.name.replace("modality_sensors/", "")] = v
restore_saver_sensors = tf.train.Saver(var_list)
with tf.variable_scope("modality_segment"):
model_emb_segment = networks.RTSN(n_seg=cfg.num_seg,
emb_dim=segment_emb_dim,
n_input=357)
model_pairsim_segment = networks.PDDM(n_input=segment_emb_dim)
input_segment_ph = tf.placeholder(tf.float32,
shape=[None, cfg.num_seg, 357])
model_emb_segment.forward(input_segment_ph, dropout_ph)
var_list = {}
for v in tf.global_variables():
if v.op.name.startswith("modality_segment"):
var_list[v.op.name.replace("modality_segment/", "")] = v
restore_saver_segment = tf.train.Saver(var_list)
############################# Forward Pass #############################
# Core branch
if cfg.normalized:
embedding = tf.nn.l2_normalize(model_emb.hidden,
axis=-1,
epsilon=1e-10)
else:
embedding = model_emb.hidden
# get the number of multimodal triplets (x3)
mul_num_ph = tf.placeholder(tf.int32, shape=[])
margins_ph = tf.placeholder(tf.float32, shape=[None])
struct_num = tf.shape(margins_ph)[0] * 3
# variable for visualizing the embeddings
emb_var = tf.Variable([0.0], name='embeddings')
set_emb = tf.assign(emb_var, embedding, validate_shape=False)
# calculated for monitoring all-pair embedding distance
diffs = utils.all_diffs_tf(embedding, embedding)
all_dist = utils.cdist_tf(diffs)
tf.summary.histogram('embedding_dists', all_dist)
# split embedding into anchor, positive and negative and calculate triplet loss
anchor, positive, negative = tf.unstack(
tf.reshape(embedding[:(tf.shape(embedding)[0] - mul_num_ph)],
[-1, 3, cfg.emb_dim]), 3, 1)
anchor_hard, positive_hard, negative_hard = tf.unstack(
tf.reshape(embedding[-mul_num_ph:-struct_num],
[-1, 3, cfg.emb_dim]), 3, 1)
anchor_struct, positive_struct, negative_struct = tf.unstack(
tf.reshape(embedding[-struct_num:], [-1, 3, cfg.emb_dim]), 3, 1)
# Sensors branch
emb_sensors = model_emb_sensors.hidden
A_sensors, B_sensors, C_sensors = tf.unstack(
tf.reshape(emb_sensors, [-1, 3, sensors_emb_dim]), 3, 1)
model_pairsim_sensors.forward(tf.stack([A_sensors, B_sensors], axis=1))
pddm_AB_sensors = model_pairsim_sensors.prob[:, 1]
model_pairsim_sensors.forward(tf.stack([A_sensors, C_sensors], axis=1))
pddm_AC_sensors = model_pairsim_sensors.prob[:, 1]
# Segment branch
emb_segment = model_emb_segment.hidden
A_segment, B_segment, C_segment = tf.unstack(
tf.reshape(emb_segment, [-1, 3, segment_emb_dim]), 3, 1)
model_pairsim_segment.forward(tf.stack([A_segment, B_segment], axis=1))
pddm_AB_segment = model_pairsim_segment.prob[:, 1]
model_pairsim_segment.forward(tf.stack([A_segment, C_segment], axis=1))
pddm_AC_segment = model_pairsim_segment.prob[:, 1]
# fuse prob from all modalities
prob_AB = 0.5 * (pddm_AB_sensors + pddm_AB_segment)
prob_AC = 0.5 * (pddm_AC_sensors + pddm_AC_segment)
############################# Calculate loss #############################
# triplet loss for labeled inputs
metric_loss1 = networks.triplet_loss(anchor, positive, negative,
cfg.alpha)
# weighted triplet loss for multimodal inputs
# if cfg.weighted:
# metric_loss2, _ = networks.weighted_triplet_loss(anchor_hard, positive_hard, negative_hard, prob_AB, prob_AC, cfg.alpha)
# else:
# triplet loss for hard examples from multimodal data
metric_loss2 = networks.triplet_loss(anchor_hard, positive_hard,
negative_hard, cfg.alpha)
# margin-based triplet loss for structure mining from multimodal data
metric_loss3 = networks.triplet_loss(anchor_struct, positive_struct,
negative_struct, margins_ph)
# whether to apply joint optimization
if cfg.no_joint:
unimodal_var_list = [
v for v in tf.global_variables()
if v.op.name.startswith("modality_core")
]
train_var_list = unimodal_var_list
else:
multimodal_var_list = [
v for v in tf.global_variables()
if not (v.op.name.startswith("modality_sensors/RTSN")
or v.op.name.startswith("modality_segment/RTSN"))
]
train_var_list = multimodal_var_list
regularization_loss = tf.reduce_sum(
tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
total_loss = tf.cond(
tf.greater(mul_num_ph, 0), lambda: tf.cond(
tf.equal(mul_num_ph,
tf.shape(embedding)[0]), lambda:
(metric_loss2 + metric_loss3 * 0.3) * cfg.lambda_multimodal +
regularization_loss * cfg.lambda_l2, lambda: metric_loss1 +
(metric_loss2 + metric_loss3 * 0.3) * cfg.lambda_multimodal +
regularization_loss * cfg.lambda_l2),
lambda: metric_loss1 + regularization_loss * cfg.lambda_l2)
tf.summary.scalar('learning_rate', lr_ph)
train_op = utils.optimize(total_loss, global_step, cfg.optimizer,
lr_ph, train_var_list)
saver = tf.train.Saver(max_to_keep=10)
summary_op = tf.summary.merge_all(
) # not logging histogram of variables because it will cause problem when only unimodal_train_op is called
summ_prob_AB = tf.summary.histogram('Prob_AB_histogram', prob_AB)
summ_prob_AC = tf.summary.histogram('Prob_AC_histogram', prob_AC)
# summ_weights = tf.summary.histogram('Weights_histogram', weights)
#########################################################################
# session iterator for session sampling
feat_paths_ph = tf.placeholder(tf.string,
shape=[None, cfg.sess_per_batch])
feat2_paths_ph = tf.placeholder(tf.string,
shape=[None, cfg.sess_per_batch])
feat3_paths_ph = tf.placeholder(tf.string,
shape=[None, cfg.sess_per_batch])
label_paths_ph = tf.placeholder(tf.string,
shape=[None, cfg.sess_per_batch])
train_data = multimodal_session_generator(
feat_paths_ph,
feat2_paths_ph,
feat3_paths_ph,
label_paths_ph,
sess_per_batch=cfg.sess_per_batch,
num_threads=2,
shuffled=False,
preprocess_func=[
model_emb.prepare_input, model_emb_sensors.prepare_input,
model_emb_segment.prepare_input
])
train_sess_iterator = train_data.make_initializable_iterator()
next_train = train_sess_iterator.get_next()
# prepare validation data
val_sess = []
val_feats = []
val_feats2 = []
val_feats3 = []
val_labels = []
val_boundaries = []
for session in val_set:
session_id = os.path.basename(session[1]).split('_')[0]
eve_batch, lab_batch, boundary = load_data_and_label(
session[0], session[-1], model_emb.prepare_input_test
) # use prepare_input_test for testing time
val_feats.append(eve_batch)
val_labels.append(lab_batch)
val_sess.extend([session_id] * eve_batch.shape[0])
val_boundaries.extend(boundary)
eve2_batch, _, _ = load_data_and_label(
session[1], session[-1], model_emb_sensors.prepare_input_test)
val_feats2.append(eve2_batch)
eve3_batch, _, _ = load_data_and_label(
session[2], session[-1], model_emb_segment.prepare_input_test)
val_feats3.append(eve3_batch)
val_feats = np.concatenate(val_feats, axis=0)
val_feats2 = np.concatenate(val_feats2, axis=0)
val_feats3 = np.concatenate(val_feats3, axis=0)
val_labels = np.concatenate(val_labels, axis=0)
print("Shape of val_feats: ", val_feats.shape)
# generate metadata.tsv for visualize embedding
with open(os.path.join(result_dir, 'metadata_val.tsv'), 'w') as fout:
fout.write('id\tlabel\tsession_id\tstart\tend\n')
for i in range(len(val_sess)):
fout.write('{0}\t{1}\t{2}\t{3}\t{4}\n'.format(
i, val_labels[i, 0], val_sess[i], val_boundaries[i][0],
val_boundaries[i][1]))
#########################################################################
# Start running the graph
if cfg.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = cfg.gpu
gpu_options = tf.GPUOptions(allow_growth=True)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
summary_writer = tf.summary.FileWriter(result_dir, sess.graph)
with sess.as_default():
sess.run(tf.global_variables_initializer())
# load pretrain model, if needed
if cfg.model_path:
print("Restoring pretrained model: %s" % cfg.model_path)
saver.restore(sess, cfg.model_path)
print("Restoring sensors model: %s" % cfg.sensors_path)
restore_saver_sensors.restore(sess, cfg.sensors_path)
print("Restoring segment model: %s" % cfg.segment_path)
restore_saver_segment.restore(sess, cfg.segment_path)
################## Training loop ##################
# Initialize pairwise embedding distance for each class on validation set
val_embeddings, _ = sess.run([embedding, set_emb],
feed_dict={
input_ph: val_feats,
dropout_ph: 1.0
})
dist_dict = {}
for i in range(np.max(val_labels) + 1):
temp_emb = val_embeddings[np.where(val_labels == i)[0]]
dist_dict[i] = [
np.mean(
utils.cdist(utils.all_diffs(temp_emb, temp_emb),
metric=cfg.metric))
]
epoch = -1
while epoch < cfg.max_epochs - 1:
step = sess.run(global_step, feed_dict=None)
epoch = step // batch_per_epoch
# learning rate schedule, reference: "In defense of Triplet Loss"
if epoch < cfg.static_epochs:
learning_rate = cfg.learning_rate
else:
learning_rate = cfg.learning_rate * \
0.01**((epoch-cfg.static_epochs)/(cfg.max_epochs-cfg.static_epochs))
# prepare data for this epoch
random.shuffle(train_set)
paths = list(zip(*[iter(train_set)] * cfg.sess_per_batch))
feat_paths = [[p[0] for p in path] for path in paths]
feat2_paths = [[p[1] for p in path] for path in paths]
feat3_paths = [[p[2] for p in path] for path in paths]
label_paths = [[p[-1] for p in path] for path in paths]
sess.run(train_sess_iterator.initializer,
feed_dict={
feat_paths_ph: feat_paths,
feat2_paths_ph: feat2_paths,
feat3_paths_ph: feat3_paths,
label_paths_ph: label_paths
})
# for each epoch
batch_count = 1
while True:
try:
##################### Data loading ########################
start_time = time.time()
eve, eve_sensors, eve_segment, lab, batch_sess = sess.run(
next_train)
# for memory concern, 1000 events are used in maximum
if eve.shape[0] > cfg.event_per_batch:
idx = np.random.permutation(
eve.shape[0])[:cfg.event_per_batch]
eve = eve[idx]
eve_sensors = eve_sensors[idx]
eve_segment = eve_segment[idx]
lab = lab[idx]
batch_sess = batch_sess[idx]
load_time = time.time() - start_time
##################### Triplet selection #####################
start_time = time.time()
# Get the embeddings of all events
eve_embedding = np.zeros((eve.shape[0], cfg.emb_dim),
dtype='float32')
for start, end in zip(
range(0, eve.shape[0], cfg.batch_size),
range(cfg.batch_size,
eve.shape[0] + cfg.batch_size,
cfg.batch_size)):
end = min(end, eve.shape[0])
emb = sess.run(embedding,
feed_dict={
input_ph: eve[start:end],
dropout_ph: 1.0
})
eve_embedding[start:end] = np.copy(emb)
# sample triplets within sampled sessions
all_diff = utils.all_diffs(eve_embedding,
eve_embedding)
triplet_selected, active_count = utils.select_triplets_facenet(
lab, utils.cdist(all_diff, metric=cfg.metric),
cfg.triplet_per_batch, cfg.alpha)
hard_count = 0
struct_count = 0
if epoch >= cfg.multimodal_epochs:
# Get the similarity of all events
sim_prob = np.zeros((eve.shape[0], eve.shape[0]),
dtype='float32') * np.nan
comb = list(
itertools.combinations(range(eve.shape[0]), 2))
for start, end in zip(
range(0, len(comb), cfg.batch_size),
range(cfg.batch_size,
len(comb) + cfg.batch_size,
cfg.batch_size)):
end = min(end, len(comb))
comb_idx = []
for c in comb[start:end]:
comb_idx.extend([c[0], c[1], c[1]])
sim = sess.run(prob_AB,
feed_dict={
input_sensors_ph:
eve_sensors[comb_idx],
input_segment_ph:
eve_segment[comb_idx],
dropout_ph:
1.0
})
for i in range(sim.shape[0]):
sim_prob[comb[start + i][0],
comb[start + i][1]] = sim[i]
sim_prob[comb[start + i][1],
comb[start + i][0]] = sim[i]
# sample triplets from similarity prediction
# maximum number not exceed the cfg.triplet_per_batch
triplet_input_idx, margins, triplet_count, hard_count, struct_count = select_triplets_mul(
triplet_selected, lab, sim_prob, dist_dict,
cfg.triplet_per_batch, 3, 0.8, 0.2)
# add up all multimodal triplets
multimodal_count = hard_count + struct_count
sensors_input = eve_sensors[
triplet_input_idx[-(3 * multimodal_count):]]
segment_input = eve_segment[
triplet_input_idx[-(3 * multimodal_count):]]
print(triplet_count, hard_count, struct_count)
triplet_input = eve[triplet_input_idx]
select_time = time.time() - start_time
if len(triplet_input.shape) > 5: # debugging
pdb.set_trace()
##################### Start training ########################
# supervised initialization
if multimodal_count == 0:
if triplet_count == 0:
continue
err, metric_err1, _, step, summ = sess.run(
[
total_loss, metric_loss1, train_op,
global_step, summary_op
],
feed_dict={
input_ph: triplet_input,
dropout_ph: cfg.keep_prob,
mul_num_ph: 0,
lr_ph: learning_rate
})
metric_err2 = 0
metric_err3 = 0
else:
err, metric_err1, metric_err2, metric_err3, _, step, summ, s_AB, s_AC = sess.run(
[
total_loss, metric_loss1, metric_loss2,
metric_loss3, train_op, global_step,
summary_op, summ_prob_AB, summ_prob_AC
],
feed_dict={
input_ph: triplet_input,
input_sensors_ph: sensors_input,
input_segment_ph: segment_input,
mul_num_ph: multimodal_count * 3,
margins_ph: margins,
dropout_ph: cfg.keep_prob,
lr_ph: learning_rate
})
summary_writer.add_summary(s_AB, step)
summary_writer.add_summary(s_AC, step)
print ("%s\tEpoch: [%d][%d/%d]\tEvent num: %d\tTriplet num: %d\tLoad time: %.3f\tSelect time: %.3f\tLoss %.4f" % \
(cfg.name, epoch+1, batch_count, batch_per_epoch, eve.shape[0], triplet_count+multimodal_count, load_time, select_time, err))
summary = tf.Summary(value=[
tf.Summary.Value(tag="train_loss",
simple_value=err),
tf.Summary.Value(tag="active_count",
simple_value=active_count),
tf.Summary.Value(tag="triplet_count",
simple_value=triplet_count),
tf.Summary.Value(tag="hard_count",
simple_value=hard_count),
tf.Summary.Value(tag="struct_count",
simple_value=struct_count),
tf.Summary.Value(tag="metric_loss1",
simple_value=metric_err1),
tf.Summary.Value(tag="metric_loss3",
simple_value=metric_err3),
tf.Summary.Value(tag="metric_loss2",
simple_value=metric_err2)
])
summary_writer.add_summary(summary, step)
summary_writer.add_summary(summ, step)
batch_count += 1
except tf.errors.OutOfRangeError:
print("Epoch %d done!" % (epoch + 1))
break
# validation on val_set
print("Evaluating on validation set...")
val_embeddings, _ = sess.run([embedding, set_emb],
feed_dict={
input_ph: val_feats,
dropout_ph: 1.0
})
mAP, mPrec, recall = utils.evaluate_simple(
val_embeddings, val_labels)
summary = tf.Summary(value=[
tf.Summary.Value(tag="Valiation mAP", simple_value=mAP),
tf.Summary.Value(tag="Validation Recall@1",
simple_value=recall),
tf.Summary.Value(tag="Validation [email protected]",
simple_value=mPrec)
])
summary_writer.add_summary(summary, step)
print("Epoch: [%d]\tmAP: %.4f\tmPrec: %.4f" %
(epoch + 1, mAP, mPrec))
# config for embedding visualization
config = projector.ProjectorConfig()
visual_embedding = config.embeddings.add()
visual_embedding.tensor_name = emb_var.name
visual_embedding.metadata_path = os.path.join(
result_dir, 'metadata_val.tsv')
projector.visualize_embeddings(summary_writer, config)
# update dist_dict
if (epoch + 1) == 50 or (epoch + 1) % 200 == 0:
for i in dist_dict.keys():
temp_emb = val_embeddings[np.where(val_labels == i)[0]]
dist_dict[i].append(
np.mean(
utils.cdist(utils.all_diffs(
temp_emb, temp_emb),
metric=cfg.metric)))
pickle.dump(
dist_dict,
open(os.path.join(result_dir, 'dist_dict.pkl'), 'wb'))
# save model
saver.save(sess,
os.path.join(result_dir, cfg.name + '.ckpt'),
global_step=step)
def select_triplets(lab,
eve_embedding,
triplet_per_batch,
alpha=0.2,
num_negative=3,
metric="squaredeuclidean"):
"""
Select the triplets for evaluation, some of them are simple, some of them are hard negative
Arguments:
eve -- array of event features, [N, n_seg, (dims)]
lab -- array of labels, [N,]
eve_embedding -- array of event embeddings, [N, emb_dim]
triplet_per_batch -- int
alpha -- float, margin
num_negative -- number of negative samples per anchor-positive pairs
metric -- metric to calculate distance
"""
# get distance for all pairs
all_diff = utils.all_diffs(eve_embedding, eve_embedding)
all_dist = utils.cdist(all_diff, metric=metric)
idx_dict = {}
for i, l in enumerate(lab):
l = int(l)
if l not in idx_dict:
idx_dict[l] = [i]
else:
idx_dict[l].append(i)
for key in idx_dict:
random.shuffle(idx_dict[key])
# create iterators for each anchor-positive pair
foreground_keys = [key for key in idx_dict.keys() if not key == 0]
foreground_dict = {}
for key in foreground_keys:
foreground_dict[key] = itertools.permutations(idx_dict[key], 2)
triplet_input_idx = []
all_neg_count = [] # for monitoring active count
while (len(triplet_input_idx)) < triplet_per_batch * 3:
keys = list(foreground_dict.keys())
if len(keys) == 0:
break
for key in keys:
try:
an_idx, pos_idx = foreground_dict[key].__next__()
except:
# remove the key to prevent infinite loop
del foreground_dict[key]
continue
pos_dist = all_dist[an_idx, pos_idx]
neg_dist = np.copy(
all_dist[an_idx]
) # important to make a copy, otherwise is reference
neg_dist[idx_dict[key]] = np.NaN
# hard ones
all_neg = np.where(
np.logical_and(neg_dist - pos_dist < alpha,
pos_dist < neg_dist))[0]
if len(all_neg) > 0:
neg_idx = all_neg[np.random.randint(len(all_neg))]
triplet_input_idx.extend([an_idx, pos_idx, neg_idx])
# simple ones
all_neg = np.where(neg_dist - pos_dist > alpha)[0]
neg_idx = all_neg[np.random.randint(len(all_neg))]
triplet_input_idx.extend([an_idx, pos_idx, neg_idx])
if len(triplet_input_idx) > 0:
return triplet_input_idx, np.mean(all_neg_count)
else:
return None, None
def main():
cfg = TrainConfig().parse()
print(cfg.name)
result_dir = os.path.join(
cfg.result_root,
cfg.name + '_' + datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S'))
if not os.path.isdir(result_dir):
os.makedirs(result_dir)
utils.write_configure_to_file(cfg, result_dir)
np.random.seed(seed=cfg.seed)
# prepare dataset
feat_train = np.load('/mnt/work/CUB_200_2011/data/feat_train.npy')
val_feats = np.load('/mnt/work/CUB_200_2011/data/feat_test.npy')
label_train = np.load('/mnt/work/CUB_200_2011/data/label_train.npy')
label_train -= 1 # make labels start from 0
val_labels = np.load('/mnt/work/CUB_200_2011/data/label_test.npy')
class_idx_dict = {}
for i, l in enumerate(label_train):
l = int(l)
if l not in class_idx_dict:
class_idx_dict[l] = [i]
else:
class_idx_dict[l].append(i)
C = len(list(class_idx_dict.keys()))
val_triplet_idx = select_triplets_random(val_labels, 1000)
# generate metadata.tsv for visualize embedding
with open(os.path.join(result_dir, 'metadata_val.tsv'), 'w') as fout:
for l in val_labels:
fout.write('{}\n'.format(int(l)))
# construct the graph
with tf.Graph().as_default():
tf.set_random_seed(cfg.seed)
global_step = tf.Variable(0, trainable=False)
lr_ph = tf.placeholder(tf.float32, name='learning_rate')
# load backbone model
model_emb = networks.CUBLayer(n_input=1024, n_output=cfg.emb_dim)
#model_emb = networks.OutputLayer(n_input=1024, n_output=cfg.emb_dim)
# get the embedding
input_ph = tf.placeholder(tf.float32, shape=[None, 1024])
dropout_ph = tf.placeholder(tf.float32, shape=[])
model_emb.forward(input_ph, dropout_ph)
if cfg.normalized:
embedding = tf.nn.l2_normalize(model_emb.logits,
axis=-1,
epsilon=1e-10)
else:
embedding = model_emb.logits
# variable for visualizing the embeddings
emb_var = tf.Variable([0.0], name='embeddings')
set_emb = tf.assign(emb_var, embedding, validate_shape=False)
# calculated for monitoring all-pair embedding distance
# diffs = utils.all_diffs_tf(embedding, embedding)
# all_dist = utils.cdist_tf(diffs)
# tf.summary.histogram('embedding_dists', all_dist)
# split embedding into anchor, positive and negative and calculate triplet loss
anchor, positive, negative = tf.unstack(
tf.reshape(embedding, [-1, 3, cfg.emb_dim]), 3, 1)
metric_loss = networks.triplet_loss(anchor, positive, negative,
cfg.alpha)
regularization_loss = tf.reduce_sum(
tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
total_loss = metric_loss + regularization_loss * cfg.lambda_l2
tf.summary.scalar('learning_rate', lr_ph)
train_op = utils.optimize(total_loss, global_step, cfg.optimizer,
lr_ph, tf.global_variables())
saver = tf.train.Saver(max_to_keep=10)
summary_op = tf.summary.merge_all()
# Start running the graph
if cfg.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = cfg.gpu
gpu_options = tf.GPUOptions(allow_growth=True)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
summary_writer = tf.summary.FileWriter(result_dir, sess.graph)
with sess.as_default():
sess.run(tf.global_variables_initializer())
################## Training loop ##################
for epoch in range(cfg.max_epochs):
# learning rate schedule, reference: "In defense of Triplet Loss"
if epoch < cfg.static_epochs:
learning_rate = cfg.learning_rate
else:
learning_rate = cfg.learning_rate * \
0.001**((epoch-cfg.static_epochs)/(cfg.max_epochs-cfg.static_epochs))
# sample images
class_in_batch = set()
idx_batch = np.array([], dtype=np.int32)
while len(idx_batch) < cfg.batch_size:
sampled_class = np.random.choice(
list(class_idx_dict.keys()))
if not sampled_class in class_in_batch:
class_in_batch.add(sampled_class)
subsample_size = np.random.choice(range(5, 11))
subsample = np.random.permutation(
class_idx_dict[sampled_class])[:subsample_size]
idx_batch = np.append(idx_batch, subsample)
idx_batch = idx_batch[:cfg.batch_size]
feat_batch = feat_train[idx_batch]
lab_batch = label_train[idx_batch]
emb = sess.run(embedding,
feed_dict={
input_ph: feat_batch,
dropout_ph: 1.0
})
# get distance for all pairs
all_diff = utils.all_diffs(emb, emb)
triplet_input_idx, active_count = select_triplets_facenet(
lab_batch,
utils.cdist(all_diff, metric=cfg.metric),
cfg.triplet_per_batch,
cfg.alpha,
num_negative=cfg.num_negative)
if triplet_input_idx is not None:
triplet_input = feat_batch[triplet_input_idx]
# perform training on the selected triplets
err, _, step, summ = sess.run(
[total_loss, train_op, global_step, summary_op],
feed_dict={
input_ph: triplet_input,
dropout_ph: cfg.keep_prob,
lr_ph: learning_rate
})
print ("%s\tEpoch: %d\tImages num: %d\tTriplet num: %d\tLoss %.4f" % \
(cfg.name, epoch+1, feat_batch.shape[0], triplet_input.shape[0]//3, err))
summary = tf.Summary(value=[
tf.Summary.Value(tag="train_loss", simple_value=err),
tf.Summary.Value(tag="active_count",
simple_value=active_count),
tf.Summary.Value(tag="images_num",
simple_value=feat_batch.shape[0]),
tf.Summary.Value(tag="triplet_num",
simple_value=triplet_input.shape[0] //
3)
])
summary_writer.add_summary(summary, step)
summary_writer.add_summary(summ, step)
# validation on val_set
if (epoch + 1) % 100 == 0:
print("Evaluating on validation set...")
val_err = sess.run(total_loss,
feed_dict={
input_ph:
val_feats[val_triplet_idx],
dropout_ph: 1.0
})
summary = tf.Summary(value=[
tf.Summary.Value(tag="Valiation loss",
simple_value=val_err),
])
print("Epoch: [%d]\tloss: %.4f" % (epoch + 1, val_err))
if (epoch + 1) % 1000 == 0:
val_embeddings, _ = sess.run([embedding, set_emb],
feed_dict={
input_ph: val_feats,
dropout_ph: 1.0
})
mAP, mPrec, recall = utils.evaluate_simple(
val_embeddings, val_labels)
summary = tf.Summary(value=[
tf.Summary.Value(tag="Valiation mAP",
simple_value=mAP),
tf.Summary.Value(tag="Validation Recall@1",
simple_value=recall),
tf.Summary.Value(tag="Validation [email protected]",
simple_value=mPrec)
])
print("Epoch: [%d]\tmAP: %.4f\trecall: %.4f" %
(epoch + 1, mAP, recall))
# config for embedding visualization
config = projector.ProjectorConfig()
visual_embedding = config.embeddings.add()
visual_embedding.tensor_name = emb_var.name
visual_embedding.metadata_path = os.path.join(
result_dir, 'metadata_val.tsv')
projector.visualize_embeddings(summary_writer, config)
summary_writer.add_summary(summary, step)
# save model
saver.save(sess,
os.path.join(result_dir, cfg.name + '.ckpt'),
global_step=step)
def main():
cfg = TrainConfig().parse()
print (cfg.name)
result_dir = os.path.join(cfg.result_root,
cfg.name+'_'+datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S'))
if not os.path.isdir(result_dir):
os.makedirs(result_dir)
utils.write_configure_to_file(cfg, result_dir)
np.random.seed(seed=cfg.seed)
# prepare dataset
train_session = cfg.train_session
train_set = prepare_dataset(cfg.feature_root, train_session, cfg.feat, cfg.label_root)
train_set = train_set[:cfg.label_num]
batch_per_epoch = len(train_set)//cfg.sess_per_batch
val_session = cfg.val_session
val_set = prepare_dataset(cfg.feature_root, val_session, cfg.feat, cfg.label_root)
# construct the graph
with tf.Graph().as_default():
tf.set_random_seed(cfg.seed)
global_step = tf.Variable(0, trainable=False)
lr_ph = tf.placeholder(tf.float32, name='learning_rate')
# load backbone model
if cfg.network == "tsn":
model_emb = networks.TSN(n_seg=cfg.num_seg, emb_dim=cfg.emb_dim)
elif cfg.network == "rtsn":
model_emb = networks.RTSN(n_seg=cfg.num_seg, emb_dim=cfg.emb_dim)
elif cfg.network == "convtsn":
model_emb = networks.ConvTSN(n_seg=cfg.num_seg, emb_dim=cfg.emb_dim)
elif cfg.network == "convrtsn":
model_emb = networks.ConvRTSN(n_seg=cfg.num_seg, emb_dim=cfg.emb_dim, n_h=cfg.n_h, n_w=cfg.n_w, n_C=cfg.n_C, n_input=cfg.n_input)
elif cfg.network == "convbirtsn":
model_emb = networks.ConvBiRTSN(n_seg=cfg.num_seg, emb_dim=cfg.emb_dim)
else:
raise NotImplementedError
# get the embedding
if cfg.feat == "sensors" or cfg.feat == "segment":
input_ph = tf.placeholder(tf.float32, shape=[None, cfg.num_seg, None])
elif cfg.feat == "resnet" or cfg.feat == "segment_down":
input_ph = tf.placeholder(tf.float32, shape=[None, cfg.num_seg, None, None, None])
dropout_ph = tf.placeholder(tf.float32, shape=[])
model_emb.forward(input_ph, dropout_ph)
if cfg.normalized:
embedding = tf.nn.l2_normalize(model_emb.hidden, axis=-1, epsilon=1e-10)
else:
embedding = model_emb.hidden
# variable for visualizing the embeddings
emb_var = tf.Variable([0.0], name='embeddings')
set_emb = tf.assign(emb_var, embedding, validate_shape=False)
# calculated for monitoring all-pair embedding distance
diffs = utils.all_diffs_tf(embedding, embedding)
all_dist = utils.cdist_tf(diffs)
tf.summary.histogram('embedding_dists', all_dist)
# split embedding into anchor, positive and negative and calculate triplet loss
anchor, positive, negative = tf.unstack(tf.reshape(embedding, [-1,3,cfg.emb_dim]), 3, 1)
metric_loss = networks.triplet_loss(anchor, positive, negative, cfg.alpha)
regularization_loss = tf.reduce_sum(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
total_loss = metric_loss + regularization_loss * cfg.lambda_l2
tf.summary.scalar('learning_rate', lr_ph)
train_op = utils.optimize(total_loss, global_step, cfg.optimizer,
lr_ph, tf.global_variables())
saver = tf.train.Saver(max_to_keep=10)
summary_op = tf.summary.merge_all()
# session iterator for session sampling
feat_paths_ph = tf.placeholder(tf.string, shape=[None, cfg.sess_per_batch])
label_paths_ph = tf.placeholder(tf.string, shape=[None, cfg.sess_per_batch])
train_data = session_generator(feat_paths_ph, label_paths_ph, sess_per_batch=cfg.sess_per_batch, num_threads=2, shuffled=False, preprocess_func=model_emb.prepare_input)
train_sess_iterator = train_data.make_initializable_iterator()
next_train = train_sess_iterator.get_next()
# prepare validation data
val_sess = []
val_feats = []
val_labels = []
val_boundaries = []
for session in val_set:
session_id = os.path.basename(session[1]).split('_')[0]
eve_batch, lab_batch, boundary = load_data_and_label(session[0], session[-1], model_emb.prepare_input_test) # use prepare_input_test for testing time
val_feats.append(eve_batch)
val_labels.append(lab_batch)
val_sess.extend([session_id]*eve_batch.shape[0])
val_boundaries.extend(boundary)
val_feats = np.concatenate(val_feats, axis=0)
val_labels = np.concatenate(val_labels, axis=0)
print ("Shape of val_feats: ", val_feats.shape)
# generate metadata.tsv for visualize embedding
with open(os.path.join(result_dir, 'metadata_val.tsv'), 'w') as fout:
fout.write('id\tlabel\tsession_id\tstart\tend\n')
for i in range(len(val_sess)):
fout.write('{0}\t{1}\t{2}\t{3}\t{4}\n'.format(i, val_labels[i,0], val_sess[i],
val_boundaries[i][0], val_boundaries[i][1]))
# Start running the graph
if cfg.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = cfg.gpu
gpu_options = tf.GPUOptions(allow_growth=True)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
summary_writer = tf.summary.FileWriter(result_dir, sess.graph)
with sess.as_default():
sess.run(tf.global_variables_initializer())
# load pretrain model, if needed
if cfg.model_path:
print ("Restoring pretrained model: %s" % cfg.model_path)
saver.restore(sess, cfg.model_path)
################## Training loop ##################
epoch = -1
while epoch < cfg.max_epochs-1:
step = sess.run(global_step, feed_dict=None)
epoch = step // batch_per_epoch
# learning rate schedule, reference: "In defense of Triplet Loss"
if epoch < cfg.static_epochs:
learning_rate = cfg.learning_rate
else:
learning_rate = cfg.learning_rate * \
0.001**((epoch-cfg.static_epochs)/(cfg.max_epochs-cfg.static_epochs))
# prepare data for this epoch
random.shuffle(train_set)
feat_paths = [path[0] for path in train_set]
label_paths = [path[1] for path in train_set]
# reshape a list to list of list
# interesting hacky code from: https://stackoverflow.com/questions/10124751/convert-a-flat-list-to-list-of-list-in-python
feat_paths = list(zip(*[iter(feat_paths)]*cfg.sess_per_batch))
label_paths = list(zip(*[iter(label_paths)]*cfg.sess_per_batch))
sess.run(train_sess_iterator.initializer, feed_dict={feat_paths_ph: feat_paths,
label_paths_ph: label_paths})
# for each epoch
batch_count = 1
while True:
try:
# Hierarchical sampling (same as fast rcnn)
start_time_select = time.time()
# First, sample sessions for a batch
eve, se, lab = sess.run(next_train)
# for memory concern, 1000 events are used in maximum
if eve.shape[0] > 1000:
idx = np.random.permutation(eve.shape[0])[:1000]
eve = eve[idx]
se = se[idx]
lab = lab[idx]
select_time1 = time.time() - start_time_select
# Get the embeddings of all events
eve_embedding = np.zeros((eve.shape[0], cfg.emb_dim), dtype='float32')
for start, end in zip(range(0, eve.shape[0], cfg.batch_size),
range(cfg.batch_size, eve.shape[0]+cfg.batch_size, cfg.batch_size)):
end = min(end, eve.shape[0])
emb = sess.run(embedding, feed_dict={input_ph: eve[start:end], dropout_ph: 1.0})
eve_embedding[start:end] = emb
# Second, sample triplets within sampled sessions
if cfg.triplet_select == 'random':
triplet_input = select_triplets_random(eve,lab,cfg.triplet_per_batch)
negative_count = 0
elif cfg.triplet_select == 'facenet':
# get distance for all pairs
all_diff = utils.all_diffs(eve_embedding, eve_embedding)
triplet_input_idx, active_count = utils.select_triplets_facenet(lab,utils.cdist(all_diff,metric=cfg.metric),cfg.triplet_per_batch,cfg.alpha,num_negative=cfg.num_negative)
else:
raise NotImplementedError
select_time2 = time.time()-start_time_select-select_time1
if (triplet_input_idx) == 0:
continue
triplet_input = eve[triplet_input_idx]
start_time_train = time.time()
# perform training on the selected triplets
err, _, step, summ = sess.run([total_loss, train_op, global_step, summary_op],
feed_dict = {input_ph: triplet_input,
dropout_ph: cfg.keep_prob,
lr_ph: learning_rate})
train_time = time.time() - start_time_train
print ("%s\tEpoch: [%d][%d/%d]\tEvent num: %d\tTriplet num: %d\tSelect_time1: %.3f\tSelect_time2: %.3f\tTrain_time: %.3f\tLoss %.4f" % \
(cfg.name, epoch+1, batch_count, batch_per_epoch, eve.shape[0], triplet_input.shape[0]//3, select_time1, select_time2, train_time, err))
summary = tf.Summary(value=[tf.Summary.Value(tag="train_loss", simple_value=err),
tf.Summary.Value(tag="active_count", simple_value=active_count),
tf.Summary.Value(tag="triplet_num", simple_value=triplet_input.shape[0]//3)])
summary_writer.add_summary(summary, step)
summary_writer.add_summary(summ, step)
batch_count += 1
except tf.errors.OutOfRangeError:
print ("Epoch %d done!" % (epoch+1))
break
# validation on val_set
print ("Evaluating on validation set...")
val_embeddings, _ = sess.run([embedding, set_emb], feed_dict={input_ph: val_feats, dropout_ph: 1.0})
mAP, mPrec, recall = utils.evaluate_simple(val_embeddings, val_labels)
summary = tf.Summary(value=[tf.Summary.Value(tag="Valiation mAP", simple_value=mAP),
tf.Summary.Value(tag="Validation Recall@1", simple_value=recall),
tf.Summary.Value(tag="Validation [email protected]", simple_value=mPrec)])
summary_writer.add_summary(summary, step)
print ("Epoch: [%d]\tmAP: %.4f\tmPrec: %.4f" % (epoch+1,mAP,mPrec))
# config for embedding visualization
config = projector.ProjectorConfig()
visual_embedding = config.embeddings.add()
visual_embedding.tensor_name = emb_var.name
visual_embedding.metadata_path = os.path.join(result_dir, 'metadata_val.tsv')
projector.visualize_embeddings(summary_writer, config)
# save model
saver.save(sess, os.path.join(result_dir, cfg.name+'.ckpt'), global_step=step)
def main():
cfg = TrainConfig().parse()
print (cfg.name)
result_dir = os.path.join(cfg.result_root,
cfg.name+'_'+datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S'))
if not os.path.isdir(result_dir):
os.makedirs(result_dir)
utils.write_configure_to_file(cfg, result_dir)
np.random.seed(seed=cfg.seed)
# prepare dataset
train_session = cfg.train_session
train_set = prepare_multimodal_dataset(cfg.feature_root, train_session, cfg.feat, cfg.label_root)
train_set = train_set[:cfg.label_num]
batch_per_epoch = len(train_set)//cfg.sess_per_batch
val_session = cfg.val_session
val_set = prepare_multimodal_dataset(cfg.feature_root, val_session, cfg.feat, cfg.label_root)
# construct the graph
with tf.Graph().as_default():
tf.set_random_seed(cfg.seed)
global_step = tf.Variable(0, trainable=False)
lr_ph = tf.placeholder(tf.float32, name='learning_rate')
####################### Load models here ########################
with tf.variable_scope("modality_core"):
# load backbone model
if cfg.network == "convtsn":
model_emb = networks.ConvTSN(n_seg=cfg.num_seg, emb_dim=cfg.emb_dim)
elif cfg.network == "convrtsn":
model_emb = networks.ConvRTSN(n_seg=cfg.num_seg, emb_dim=cfg.emb_dim)
else:
raise NotImplementedError
input_ph = tf.placeholder(tf.float32, shape=[None, cfg.num_seg, None, None, None])
dropout_ph = tf.placeholder(tf.float32, shape=[])
model_emb.forward(input_ph, dropout_ph) # for lstm has variable scope
with tf.variable_scope("modality_sensors"):
sensors_emb_dim = 32
model_emb_sensors = networks.RTSN(n_seg=cfg.num_seg, emb_dim=sensors_emb_dim)
input_sensors_ph = tf.placeholder(tf.float32, shape=[None, cfg.num_seg, 8])
model_emb_sensors.forward(input_sensors_ph, dropout_ph)
var_list = {}
for v in tf.global_variables():
if v.op.name.startswith("modality_sensors"):
var_list[v.op.name.replace("modality_sensors/","")] = v
restore_saver_sensors = tf.train.Saver(var_list)
with tf.variable_scope("modality_segment"):
segment_emb_dim = 32
model_emb_segment = networks.RTSN(n_seg=cfg.num_seg, emb_dim=segment_emb_dim, n_input=357)
input_segment_ph = tf.placeholder(tf.float32, shape=[None, cfg.num_seg, 357])
model_emb_segment.forward(input_segment_ph, dropout_ph)
var_list = {}
for v in tf.global_variables():
if v.op.name.startswith("modality_segment"):
var_list[v.op.name.replace("modality_segment/","")] = v
restore_saver_segment = tf.train.Saver(var_list)
############################# Forward Pass #############################
# Core branch
if cfg.normalized:
embedding = tf.nn.l2_normalize(model_emb.hidden, axis=-1, epsilon=1e-10)
embedding_sensors = tf.nn.l2_normalize(model_emb_sensors.hidden, axis=-1, epsilon=1e-10)
embedding_hal_sensors = tf.nn.l2_normalize(hal_emb_sensors.hidden, axis=-1, epsilon=1e-10)
embedding_segment = tf.nn.l2_normalize(model_emb_segment.hidden, axis=-1, epsilon=1e-10)
embedding_hal_segment = tf.nn.l2_normalize(hal_emb_segment.hidden, axis=-1, epsilon=1e-10)
else:
embedding = model_emb.hidden
embedding_sensors = model_emb_sensors.hidden
embedding_hal_sensors = hal_emb_sensors.hidden
embedding_segment = model_emb_segment.hidden
embedding_hal_segment = hal_emb_segment.hidden
# variable for visualizing the embeddings
emb_var = tf.Variable([0.0], name='embeddings')
set_emb = tf.assign(emb_var, embedding, validate_shape=False)
# calculated for monitoring all-pair embedding distance
diffs = utils.all_diffs_tf(embedding, embedding)
all_dist = utils.cdist_tf(diffs)
tf.summary.histogram('embedding_dists', all_dist)
# split embedding into anchor, positive and negative and calculate triplet loss
anchor, positive, negative = tf.unstack(tf.reshape(embedding, [-1,3,cfg.emb_dim]), 3, 1)
anc_sensors, pos_sensors, neg_sensors = tf.unstack(tf.reshape(embedding_sensors, [-1,3,sensors_emb_dim]), 3, 1)
anc_hal_sensors, pos_hal_sensors, neg_hal_sensors = tf.unstack(tf.reshape(embedding_hal_sensors, [-1,3,sensors_emb_dim]), 3, 1)
anc_segment, pos_segment, neg_segment = tf.unstack(tf.reshape(embedding_segment, [-1,3,segment_emb_dim]), 3, 1)
anc_hal_segment, pos_hal_segment, neg_hal_segment = tf.unstack(tf.reshape(embedding_hal_segment, [-1,3,segment_emb_dim]), 3, 1)
# a fusion embedding
anc_fused = tf.concat((anchor, anc_hal_sensors, anc_hal_segment), axis=1)
pos_fused = tf.concat((positive, pos_hal_sensors, anc_hal_segment), axis=1)
neg_fused = tf.concat((negative, neg_hal_sensors, anc_hal_segment), axis=1)
############################# Calculate loss #############################
# triplet loss
metric_loss = networks.triplet_loss(anchor, positive, negative, cfg.alpha) + \
networks.triplet_loss(anc_sensors, pos_sensors, neg_sensors, cfg.alpha) + \
networks.triplet_loss(anc_hal_sensors, pos_hal_sensors, neg_hal_sensors, cfg.alpha) + \
networks.triplet_loss(anc_segment, pos_segment, neg_segment, cfg.alpha) + \
networks.triplet_loss(anc_hal_segment, pos_hal_segment, neg_hal_segment, cfg.alpha) + \
networks.triplet_loss(anc_fused, pos_fused, neg_fused, cfg.alpha)
# hallucination loss (regression loss)
hal_loss_sensors = tf.nn.l2_loss(embedding_sensors - embedding_hal_sensors)
hal_loss_segment = tf.nn.l2_loss(embedding_segment - embedding_hal_segment)
hal_loss = hal_loss_sensors + hal_loss_segment
regularization_loss = tf.reduce_sum(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
# use lambda_multimodal for hal_loss
total_loss = metric_loss + cfg.lambda_multimodal * hal_loss + regularization_loss * cfg.lambda_l2
tf.summary.scalar('learning_rate', lr_ph)
train_op = utils.optimize(total_loss, global_step, cfg.optimizer,
lr_ph, tf.global_variables())
saver = tf.train.Saver(max_to_keep=10)
summary_op = tf.summary.merge_all() # not logging histogram of variables because it will cause problem when only unimodal_train_op is called
#########################################################################
# session iterator for session sampling
feat_paths_ph = tf.placeholder(tf.string, shape=[None, cfg.sess_per_batch])
feat2_paths_ph = tf.placeholder(tf.string, shape=[None, cfg.sess_per_batch])
feat3_paths_ph = tf.placeholder(tf.string, shape=[None, cfg.sess_per_batch])
label_paths_ph = tf.placeholder(tf.string, shape=[None, cfg.sess_per_batch])
train_data = multimodal_session_generator(feat_paths_ph, feat2_paths_ph, feat3_paths_ph, label_paths_ph, sess_per_batch=cfg.sess_per_batch, num_threads=2, shuffled=False, preprocess_func=[model_emb.prepare_input, model_emb_sensors.prepare_input, model_emb_segment.prepare_input])
train_sess_iterator = train_data.make_initializable_iterator()
next_train = train_sess_iterator.get_next()
# prepare validation data
val_sess = []
val_feats = []
val_feats2 = []
val_feats3 = []
val_labels = []
val_boundaries = []
for session in val_set:
session_id = os.path.basename(session[1]).split('_')[0]
eve_batch, lab_batch, boundary = load_data_and_label(session[0], session[-1], model_emb.prepare_input_test) # use prepare_input_test for testing time
val_feats.append(eve_batch)
val_labels.append(lab_batch)
val_sess.extend([session_id]*eve_batch.shape[0])
val_boundaries.extend(boundary)
eve2_batch, _,_ = load_data_and_label(session[1], session[-1], model_emb_sensors.prepare_input_test)
val_feats2.append(eve2_batch)
eve3_batch, _,_ = load_data_and_label(session[2], session[-1], model_emb_segment.prepare_input_test)
val_feats3.append(eve3_batch)
val_feats = np.concatenate(val_feats, axis=0)
val_feats2 = np.concatenate(val_feats2, axis=0)
val_feats3 = np.concatenate(val_feats3, axis=0)
val_labels = np.concatenate(val_labels, axis=0)
print ("Shape of val_feats: ", val_feats.shape)
# generate metadata.tsv for visualize embedding
with open(os.path.join(result_dir, 'metadata_val.tsv'), 'w') as fout:
fout.write('id\tlabel\tsession_id\tstart\tend\n')
for i in range(len(val_sess)):
fout.write('{0}\t{1}\t{2}\t{3}\t{4}\n'.format(i, val_labels[i,0], val_sess[i],
val_boundaries[i][0], val_boundaries[i][1]))
#########################################################################
# Start running the graph
if cfg.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = cfg.gpu
gpu_options = tf.GPUOptions(allow_growth=True)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
summary_writer = tf.summary.FileWriter(result_dir, sess.graph)
with sess.as_default():
sess.run(tf.global_variables_initializer())
print ("Restoring sensors model: %s" % cfg.sensors_path)
restore_saver_sensors.restore(sess, cfg.sensors_path)
print ("Restoring segment model: %s" % cfg.segment_path)
restore_saver_segment.restore(sess, cfg.segment_path)
# load pretrain model, if needed
if cfg.model_path:
print ("Restoring pretrained model: %s" % cfg.model_path)
saver.restore(sess, cfg.model_path)
################## Training loop ##################
epoch = -1
while epoch < cfg.max_epochs-1:
step = sess.run(global_step, feed_dict=None)
epoch = step // batch_per_epoch
# learning rate schedule, reference: "In defense of Triplet Loss"
if epoch < cfg.static_epochs:
learning_rate = cfg.learning_rate
else:
learning_rate = cfg.learning_rate * \
0.01**((epoch-cfg.static_epochs)/(cfg.max_epochs-cfg.static_epochs))
# prepare data for this epoch
random.shuffle(train_set)
paths = list(zip(*[iter(train_set)]*cfg.sess_per_batch))
feat_paths = [[p[0] for p in path] for path in paths]
feat2_paths = [[p[1] for p in path] for path in paths]
feat3_paths = [[p[2] for p in path] for path in paths]
label_paths = [[p[-1] for p in path] for path in paths]
sess.run(train_sess_iterator.initializer, feed_dict={feat_paths_ph: feat_paths,
feat2_paths_ph: feat2_paths,
feat3_paths_ph: feat3_paths,
label_paths_ph: label_paths})
# for each epoch
batch_count = 1
while True:
try:
##################### Data loading ########################
start_time = time.time()
eve, eve_sensors, eve_segment, lab, batch_sess = sess.run(next_train)
load_time = time.time() - start_time
##################### Triplet selection #####################
start_time = time.time()
# Get the embeddings of all events
eve_embedding = np.zeros((eve.shape[0], cfg.emb_dim), dtype='float32')
for start, end in zip(range(0, eve.shape[0], cfg.batch_size),
range(cfg.batch_size, eve.shape[0]+cfg.batch_size, cfg.batch_size)):
end = min(end, eve.shape[0])
emb = sess.run(embedding, feed_dict={input_ph: eve[start:end], dropout_ph: 1.0})
eve_embedding[start:end] = np.copy(emb)
# sample triplets within sampled sessions
all_diff = utils.all_diffs(eve_embedding, eve_embedding)
triplet_input_idx, active_count = utils.select_triplets_facenet(lab,utils.cdist(all_diff,metric=cfg.metric),cfg.triplet_per_batch,cfg.alpha,num_negative=cfg.num_negative)
if triplet_input_idx is None:
continue
triplet_input = eve[triplet_input_idx]
sensors_input = eve_sensors[triplet_input_idx]
segment_input = eve_segment[triplet_input_idx]
select_time = time.time() - start_time
if len(triplet_input.shape) > 5: # debugging
pdb.set_trace()
##################### Start training ########################
err, metric_err, hal_err, _, step, summ = sess.run(
[total_loss, metric_loss, hal_loss, train_op, global_step, summary_op],
feed_dict = {input_ph: triplet_input,
input_sensors_ph: sensors_input,
input_segment_ph: segment_input,
dropout_ph: cfg.keep_prob,
lr_ph: learning_rate})
print ("%s\tEpoch: [%d][%d/%d]\tEvent num: %d\tTriplet num: %d\tLoad time: %.3f\tSelect time: %.3f\tMetric Loss %.4f\tHal Loss %.4f" % \
(cfg.name, epoch+1, batch_count, batch_per_epoch, eve.shape[0], triplet_input.shape[0]//3, load_time, select_time, metric_err, hal_err))
summary = tf.Summary(value=[tf.Summary.Value(tag="train_loss", simple_value=err),
tf.Summary.Value(tag="active_count", simple_value=active_count),
tf.Summary.Value(tag="metric_loss", simple_value=metric_err),
tf.Summary.Value(tag="hallucination_loss", simple_value=hal_err)])
summary_writer.add_summary(summary, step)
summary_writer.add_summary(summ, step)
batch_count += 1
except tf.errors.OutOfRangeError:
print ("Epoch %d done!" % (epoch+1))
break
# validation on val_set
print ("Evaluating on validation set...")
val_embeddings, hal_err, _ = sess.run([embedding, hal_loss, set_emb],
feed_dict = {input_ph: val_feats,
input_sensors_ph: val_feats2,
input_segment_ph: val_feats3,
dropout_ph: 1.0})
mAP, mPrec = utils.evaluate_simple(val_embeddings, val_labels)
summary = tf.Summary(value=[tf.Summary.Value(tag="Valiation mAP", simple_value=mAP),
tf.Summary.Value(tag="Validation [email protected]", simple_value=mPrec),
tf.Summary.Value(tag="Validation hal loss", simple_value=hal_err)])
summary_writer.add_summary(summary, step)
print ("Epoch: [%d]\tmAP: %.4f\tmPrec: %.4f" % (epoch+1,mAP,mPrec))
# config for embedding visualization
config = projector.ProjectorConfig()
visual_embedding = config.embeddings.add()
visual_embedding.tensor_name = emb_var.name
visual_embedding.metadata_path = os.path.join(result_dir, 'metadata_val.tsv')
projector.visualize_embeddings(summary_writer, config)
# save model
saver.save(sess, os.path.join(result_dir, cfg.name+'.ckpt'), global_step=step)
def select_triplets_facenet(lab,
eve_embedding,
triplet_per_batch,
alpha=0.2,
num_negative=3,
metric="squaredeuclidean"):
"""
Select the triplets for training
1. Sample anchor-positive pair (try to balance imbalanced classes)
2. Semi-hard negative mining used in facenet
Arguments:
lab -- array of labels, [N,]
eve_embedding -- array of event embeddings, [N, emb_dim]
triplet_per_batch -- int
alpha -- float, margin
num_negative -- number of negative samples per anchor-positive pairs
metric -- metric to calculate distance
"""
# get distance for all pairs
all_diff = utils.all_diffs(eve_embedding, eve_embedding)
all_dist = utils.cdist(all_diff, metric=metric)
idx_dict = {}
for i, l in enumerate(lab):
l = int(l)
if l not in idx_dict:
idx_dict[l] = [i]
else:
idx_dict[l].append(i)
for key in idx_dict:
random.shuffle(idx_dict[key])
# create iterators for each anchor-positive pair
foreground_keys = [key for key in idx_dict.keys() if not key == 0]
foreground_dict = {}
for key in foreground_keys:
foreground_dict[key] = itertools.permutations(idx_dict[key], 2)
triplet_input_idx = []
all_neg_count = [] # for monitoring active count
while (len(triplet_input_idx)) < triplet_per_batch * 3:
keys = list(foreground_dict.keys())
if len(keys) == 0:
break
for key in keys:
try:
an_idx, pos_idx = foreground_dict[key].__next__()
except:
# remove the key to prevent infinite loop
del foreground_dict[key]
continue
pos_dist = all_dist[an_idx, pos_idx]
neg_dist = np.copy(
all_dist[an_idx]
) # important to make a copy, otherwise is reference
neg_dist[idx_dict[key]] = np.NaN
all_neg = np.where(
np.logical_and(neg_dist - pos_dist < alpha,
pos_dist < neg_dist))[0]
all_neg_count.append(len(all_neg))
# continue if no proper negtive sample
if len(all_neg) > 0:
for i in range(num_negative):
neg_idx = all_neg[np.random.randint(len(all_neg))]
triplet_input_idx.extend([an_idx, pos_idx, neg_idx])
#triplet_input.append(np.expand_dims(eve[an_idx],0))
#triplet_input.append(np.expand_dims(eve[pos_idx],0))
#triplet_input.append(np.expand_dims(eve[neg_idx],0))
if len(triplet_input) > 0:
return triplet_input_idx, np.mean(all_neg_count)
# return np.concatenate(triplet_input, axis=0), np.mean(all_neg_count)
else:
return None, None
def forward(self, prob_map, gt, orig_sizes):
"""
Compute the Weighted Hausdorff Distance function
between the estimated probability map and ground truth points.
The output is the WHD averaged through all the batch.
:param prob_map: (B x H x W) Tensor of the probability map of the estimation.
B is batch size, H is height and W is width.
Values must be between 0 and 1.
:param gt: List of Tensors of the Ground Truth points.
Must be of size B as in prob_map.
Each element in the list must be a 2D Tensor,
where each row is the (y, x), i.e, (row, col) of a GT point.
:param orig_sizes: Bx2 Tensor containing the size
of the original images.
B is batch size.
The size must be in (height, width) format.
:param orig_widths: List of the original widths for each image
in the batch.
:return: Single-scalar Tensor with the Weighted Hausdorff Distance.
If self.return_2_terms=True, then return a tuple containing
the two terms of the Weighted Hausdorff Distance.
"""
_assert_no_grad([gt])
assert prob_map.dim() == 3, 'The probability map must be (B x H x W)'
assert prob_map.size()[1:3] == (self.height, self.width), \
'You must configure the WeightedHausdorffDistance with the height and width of the ' \
'probability map that you are using, got a probability map of size %s'\
% str(prob_map.size())
batch_size = prob_map.shape[0]
assert batch_size == gt.shape[0]
self.all_img_locations = self.all_img_locations.to(prob_map.device)
self.resized_size = self.resized_size.to(prob_map.device)
terms_1 = []
terms_2 = []
for b in range(batch_size):
# One by one
prob_map_b = prob_map[b, :, :]
gt_b = gt[b, :].unsqueeze(0) # Ensure point is [1, 2]
orig_size_b = orig_sizes[b, :]
norm_factor = (orig_size_b / self.resized_size).unsqueeze(0)
n_gt_pts = gt_b.size()[0]
# Corner case: no GT points
if gt_b.ndimension() == 1 and (gt_b < 0).all().item() == 0:
terms_1.append(
torch.tensor([0], dtype=torch.get_default_dtype()))
terms_2.append(
torch.tensor([self.max_dist],
dtype=torch.get_default_dtype()))
continue
# Pairwise distances between all possible locations and the GTed locations
n_gt_pts = gt_b.size()[0]
normalized_x = norm_factor.repeat(self.n_pixels,
1) * self.all_img_locations
normalized_y = norm_factor.repeat(len(gt_b), 1) * gt_b
d_matrix = cdist(normalized_x, normalized_y)
# Reshape probability map as a long column vector,
# and prepare it for multiplication
p = prob_map_b.view(prob_map_b.nelement())
n_est_pts = p.sum()
p_replicated = p.view(-1, 1).repeat(1, n_gt_pts)
# Weighted Hausdorff Distance
term_1 = (1 / (n_est_pts + 1e-6)) * torch.sum(
p * torch.min(d_matrix, 1)[0])
weighted_d_matrix = (
1 - p_replicated) * self.max_dist + p_replicated * d_matrix
minn = generaliz_mean(weighted_d_matrix,
p=self.p,
dim=0,
keepdim=False)
term_2 = torch.mean(minn)
# terms_1[b] = term_1
# terms_2[b] = term_2
terms_1.append(term_1)
terms_2.append(term_2)
terms_1 = torch.stack(terms_1)
terms_2 = torch.stack(terms_2)
if self.return_2_terms:
res = terms_1, terms_2
else:
res = terms_1 + terms_2
return res