def infer(self, images, do_flip=False):
"""
:param images: utils.py-->load_data
rgb format
resize to (image_height, image_width, 3)
img = img - 127.5
img = img / 128.
:return:
"""
feed_dict = {
self.images_placeholder: images,
self.phase_train_placeholder: False
}
feats = self.__session.run(self.embeddings, feed_dict=feed_dict)
if do_flip:
images_flip = [np.fliplr(image) for image in images]
feed_dict_flip = {
self.images_placeholder: images_flip,
self.phase_train_placeholder: False
}
feats_flip = self.__session.run(self.embeddings,
feed_dict=feed_dict_flip)
feats = np.concatenate((feats, feats_flip), axis=1)
feats = utils.l2_normalize(feats)
return feats
def forward(self, x, target, y, idx=None):
'''
x: featrues
y: indices
'''
K = int(self.params[0].item())
T = self.params[1].item()
Z = self.params[2].item()
momentum = self.params[3].item()
z_momentum = self.params[4].item()
# pdb.set_trace()
bs = x.size(0)
# nce_core
nce_core_out = self.nce_core(x, y, idx, K)
# NN statistics
out = torch.mm(x, self.memory.t()) # (bs, dim) (dim, num)
yd, yi = out.topk(32, dim=1, largest=True, sorted=True)
candidates = self.trainLabel.view(1, -1).expand(bs, -1)
retrieval = torch.gather(candidates, 1, yi)
show_distribution(yd.cpu(), retrieval.cpu(), target.cpu())
pdb.set_trace()
out = self.nce_core_ms(out, K)
if self.use_softmax:
out = torch.div(out, T)
out = out.squeeze().contiguous()
probs = self.extract_probs(out)
else:
out = torch.exp(torch.div(out, T))
if Z < 0:
# initialize it with mean of first batch
self.params[2] = out.mean() * self.outputSize
Z = self.params[2].item()
print('normalization constant Z is set to {:.1f}'.format(Z))
# else:
# # update normalization factor with z_momentum
# Z_new = out.mean() * self.outputSize
# self.params[2] = z_momentum * Z_new + (1 - z_momentum) * self.params[2]
# Z = self.params[2].clone().detach().item()
out = torch.div(out, Z).squeeze().contiguous()
probs = self.extract_probs(out)
# update memory
with torch.no_grad():
weight_pos = torch.index_select(self.memory, 0, y.view(-1))
weight_pos.mul_(momentum)
weight_pos.add_(torch.mul(x, 1 - momentum))
# l2 normalize it again
normed_weight = l2_normalize(weight_pos)
self.memory.index_copy_(0, y, normed_weight)
return out, probs
def update_new_data_memory(self, idxs):
data_memory = torch.index_select(self.memory, 0, idxs)
# pdb.set_trace()
# use nearest neighbour's mean to update memory bank
new_data_memory = torch.mean(self.memory[self.yi], dim=1)
new_data_memory = new_data_memory.view(self.clips_num, self.batch_size, self.embed_dim).permute(1, 0, 2)
# use each clip's neighbours' mean to update
new_data_memory = torch.mean(new_data_memory, dim=1)
new_data_memory = data_memory * self.params[3] + (1 - self.params[3]) * new_data_memory
new_data_memory = l2_normalize(new_data_memory)
idxs = idxs.unsqueeze(1).repeat(1, self.embed_dim)
self.memory.scatter_(0, idxs, new_data_memory)
def get_normal_vector(model, train_normal_loader_for_test, cal_vec_batch_size,
feature_dim, use_cuda):
total_batch = int(len(train_normal_loader_for_test))
print(
"=====================================Calculating Average Normal Vector====================================="
)
if use_cuda:
normal_vec = torch.zeros((1, 512)).cuda()
else:
normal_vec = torch.zeros((1, 512))
for batch, (normal_data, idx) in enumerate(train_normal_loader_for_test):
if use_cuda:
normal_data = normal_data.cuda()
_, outputs = model(normal_data)
outputs = outputs.detach()
normal_vec = (torch.sum(outputs, dim=0) +
normal_vec * batch * cal_vec_batch_size) / (
(batch + 1) * cal_vec_batch_size)
print(
f'Calculating Average Normal Vector: Batch {batch + 1} / {total_batch}'
)
normal_vec = l2_normalize(normal_vec)
return normal_vec
def at(x):
q = x.pow(2).mean(1).view(x.size(0), -1)
return l2_normalize(q)
for epoch in range(begin_epoch, begin_epoch + args.epochs + 1):
memory_bank, loss = train(train_normal_loader,
train_anormal_loader, model, model_head,
nce_average, criterion, optimizer, epoch,
args, batch_logger, epoch_logger,
memory_bank)
if epoch % args.val_step == 0:
print(
"==========================================!!!Evaluating!!!=========================================="
)
normal_vec = torch.mean(torch.cat(memory_bank, dim=0),
dim=0,
keepdim=True)
normal_vec = l2_normalize(normal_vec)
model.eval()
accuracy, best_threshold, acc_n, acc_a, acc_list, acc_n_list, acc_a_list = split_acc_diff_threshold(
model, normal_vec, validation_loader, args.use_cuda)
print(
f'Epoch: {epoch}/{args.epochs} | Accuracy: {accuracy} | Normal Acc: {acc_n} | Anormal Acc: {acc_a} | Threshold: {best_threshold}'
)
print(
"==========================================!!!Logging!!!=========================================="
)
val_logger.log({
'epoch': epoch,
'accuracy': accuracy * 100,
'normal_acc': acc_n * 100,
'anormal_acc': acc_a * 100,
# Assume noise distribution is a uniform distribution
q_noise = 1. / self.num_data
# P(1|normal_v, vi) = p_p / (p_p + k*q_noise)
p_p = x.select(1, 0) # equal to x[:, 0] and p_p is p(vi|normal_v)
log_D1 = torch.div(p_p, p_p.add(k * q_noise + eps)).log_()
# Second term of NCE Loss which is loss for negative pairs
# P(0|normal_v, vi_prime) = P(origin=noise) = k*q_noise / (p_n + k*q_noise)
p_n = x.narrow(1, 1, k) # narrow(dim, start, len) equal to x[:, 1:K+1] and p_n is p(vi_prime|normal_v)
log_D0 = torch.div(p_n.clone().fill_(k*q_noise), p_n.add(k*q_noise+eps)).log_() # clone is just to get a same size matrix and be filled with k*q_noise
loss = -(log_D1.sum(0) + log_D0.view(-1, 1).sum(0)) / batch_size
return loss
if __name__ == '__main__':
average = NCEAverage(128, 9000, 0.07, 28, 0.9).cuda()
criterion = NCECriterion(9000)
dummy_n_embeddings = torch.randn(4, 128).cuda()
dummy_a_embeddings = torch.randn(28, 128).cuda()
dummy_n_embeddings = l2_normalize(dummy_n_embeddings)
dummy_a_embeddings = l2_normalize(dummy_a_embeddings)
outs, probs = average(dummy_n_embeddings, dummy_a_embeddings)
print(outs)
print(outs.shape)
loss = criterion(outs)
print(loss)
self.embeddings = x
# treat the last batch specially
if i == self.loader_length - 1:
self.prepare_indices(batch_size, bs)
pos_logits = self.compute_data_prob()
neg_logits = self.compute_noise_prob()
outs, probs = self.nce_core(pos_logits, neg_logits)
# pdb.set_trace()
with torch.no_grad():
self.update_new_data_memory(idxs)
return outs, probs
if __name__ == '__main__':
c_average = ClusterAverage(128, 9000, 239, 4, 0.07, 3).cuda()
print(c_average.neg_indices)
n_average = NCEAverage(128, 9000, 239, 4, 0.07, 3).cuda()
# initialize c_average with n_average
c_average.init_memoryBank(n_average)
dummy_embeddings = torch.randn(12, 128).cuda()
dummy_embeddings = l2_normalize(dummy_embeddings)
idxs = torch.arange(3).cuda()
outs_c, probs_c = c_average(dummy_embeddings, idxs, 0)
outs_n, probs_n, _ = n_average(dummy_embeddings, idxs, 0)
pdb.set_trace()
def main(args):
with tf.Graph().as_default():
with tf.Session() as sess:
# Read the file containing the pairs used for testing
pairs = lfw.read_pairs(os.path.expanduser(args.lfw_pairs))
#pdb.set_trace()
# Get the paths for the corresponding images
paths, actual_issame = lfw.get_paths(
os.path.expanduser(args.lfw_dir), pairs, args.lfw_file_ext)
# Load the model
#facenet.load_model(args.model)
# Get input and output tensors
#image_size = images_placeholder.get_shape()[1] # For some reason this doesn't work for frozen graphs
image_size = args.image_size
print('image size', image_size)
#images_placeholder = tf.placeholder(tf.float32,shape=(None,image_size,image_size,3),name='image')
images_placeholder = tf.placeholder(tf.float32,
shape=(None, args.image_height,
args.image_width, 3),
name='image')
phase_train_placeholder = tf.placeholder(tf.bool,
name='phase_train')
#with slim.arg_scope(resnet_v1.resnet_arg_scope(False)):
if args.network_type == 'resnet50':
with slim.arg_scope(resnet_v2.resnet_arg_scope(False)):
prelogits, end_points = resnet_v2.resnet_v2_50(
images_placeholder,
is_training=phase_train_placeholder,
num_classes=256,
output_stride=16)
#prelogits, end_points = resnet_v2.resnet_v2_50(images_placeholder,is_training=phase_train_placeholder,num_classes=256,output_stride=8)
#prelogits, end_points = resnet_v2_modify.resnet_v2_50(images_placeholder,is_training=phase_train_placeholder,num_classes=256)
#prelogits = slim.batch_norm(prelogits, is_training=phase_train_placeholder,epsilon=1e-5, scale=True,scope='softmax_bn')
prelogits = tf.squeeze(prelogits, [1, 2],
name='SpatialSqueeze')
elif args.network_type == 'sphere_network':
prelogits = network.infer(images_placeholder)
if args.fc_bn:
prelogits = slim.batch_norm(
prelogits,
is_training=phase_train_placeholder,
epsilon=1e-5,
scale=True,
scope='softmax_bn')
#embeddings = tf.nn.l2_normalize(prelogits, 1, 1e-10, name='embeddings')
embeddings = tf.identity(prelogits)
#saver = tf.train.Saver(tf.trainable_variables(), max_to_keep=3)
saver = tf.train.Saver(tf.global_variables(), max_to_keep=3)
saver.restore(sess, args.model)
if args.save_model:
saver.save(sess, './tmp_saved_model', global_step=1)
return 0
embedding_size = embeddings.get_shape()[1]
# Run forward pass to calculate embeddings
print('Runnning forward pass on LFW images')
batch_size = args.lfw_batch_size
nrof_images = len(paths)
nrof_batches = int(math.ceil(1.0 * nrof_images / batch_size))
if args.do_flip:
embedding_size *= 2
emb_array = np.zeros((nrof_images, embedding_size))
else:
emb_array = np.zeros((nrof_images, embedding_size))
for i in range(nrof_batches):
start_index = i * batch_size
print('handing {}/{}'.format(start_index, nrof_images))
end_index = min((i + 1) * batch_size, nrof_images)
paths_batch = paths[start_index:end_index]
#images = facenet.load_data(paths_batch, False, False, image_size,True,image_size)
#images = facenet.load_data2(paths_batch, False, False, args.image_height,args.image_width,True,)
images = utils.load_data(paths_batch, False, True,
args.image_height, args.image_width,
True,
(args.image_height, args.image_width))
feed_dict = {
images_placeholder: images,
phase_train_placeholder: False
}
feats = sess.run(embeddings, feed_dict=feed_dict)
if args.do_flip:
images_flip = utils.load_data(
paths_batch, False, True, args.image_height,
args.image_width, True,
(args.image_height, args.image_width))
feed_dict = {
images_placeholder: images_flip,
phase_train_placeholder: False
}
feats_flip = sess.run(embeddings, feed_dict=feed_dict)
feats = np.concatenate((feats, feats_flip), axis=1)
#feats = (feats+feats_flip)/2
#images = facenet.load_data(paths_batch, False, False, 160,True,182)
#images = facenet.load_data(paths_batch, False, False, image_size,src_size=256)
#feed_dict = { images_placeholder:images, phase_train_placeholder:True}
#pdb.set_trace()
#feats = facenet.prewhiten(feats)
feats = utils.l2_normalize(feats)
emb_array[start_index:end_index, :] = feats
#pdb.set_trace()
tpr, fpr, accuracy, val, val_std, far = lfw.evaluate(
emb_array, actual_issame, nrof_folds=args.lfw_nrof_folds)
print('Accuracy: %1.3f+-%1.3f' %
(np.mean(accuracy), np.std(accuracy)))
print('Validation rate: %2.5f+-%2.5f @ FAR=%2.5f' %
(val, val_std, far))
auc = metrics.auc(fpr, tpr)
print('Area Under Curve (AUC): %1.3f' % auc)
eer = brentq(lambda x: 1. - x - interpolate.interp1d(fpr, tpr)(x),
0., 1.)
print('Equal Error Rate (EER): %1.3f' % eer)
def evaluate():
model.eval()
# 语料向量化
all_vecs = []
for a_texts, b_texts in all_texts:
a_data = SimCSE_DataSet(a_texts, tokenizer, max_len)
a_data_gen = DataLoader(a_data,
batch_size=args.train_batch_size,
collate_fn=collate_func)
b_data = SimCSE_DataSet(b_texts, tokenizer, max_len)
b_data_gen = DataLoader(b_data,
batch_size=args.train_batch_size,
collate_fn=collate_func)
all_a_vecs = []
for eval_batch in tqdm(a_data_gen):
if torch.cuda.is_available():
input_ids = eval_batch["input_ids"].cuda()
attention_mask_ids = eval_batch["attention_mask_ids"].cuda()
else:
input_ids = eval_batch["input_ids"]
attention_mask_ids = eval_batch["attention_mask_ids"]
with torch.no_grad():
eval_encodings = model(input_ids=input_ids,
attention_mask=attention_mask_ids)
eval_encodings = eval_encodings.cpu().detach().numpy()
all_a_vecs.extend(eval_encodings)
all_b_vecs = []
for eval_batch in tqdm(b_data_gen):
if torch.cuda.is_available():
input_ids = eval_batch["input_ids"].cuda()
attention_mask_ids = eval_batch["attention_mask_ids"].cuda()
else:
input_ids = eval_batch["input_ids"]
attention_mask_ids = eval_batch["attention_mask_ids"]
with torch.no_grad():
eval_encodings = model(input_ids=input_ids,
attention_mask=attention_mask_ids)
eval_encodings = eval_encodings.cpu().detach().numpy()
all_b_vecs.extend(eval_encodings)
all_vecs.append((np.array(all_a_vecs), np.array(all_b_vecs)))
# 标准化，相似度，相关系数
all_corrcoefs = []
for (a_vecs, b_vecs), labels in zip(all_vecs, all_labels):
a_vecs = l2_normalize(a_vecs)
b_vecs = l2_normalize(b_vecs)
sims = (a_vecs * b_vecs).sum(axis=1)
corrcoef = compute_corrcoef(labels, sims)
all_corrcoefs.append(corrcoef)
all_corrcoefs.extend([
np.average(all_corrcoefs),
np.average(all_corrcoefs, weights=all_weights)
])
print(all_corrcoefs)
return all_corrcoefs
def extract_features(model, source, destination, image_height, image_width,
prewhiten, fc_bn, feature_size):
if path.isfile(source):
full_path = True
source_list = np.sort(np.loadtxt(source, dtype=np.str))
else:
full_path = False
source_list = listdir(source)
with tf.Graph().as_default():
with tf.Session() as sess:
images_placeholder = tf.placeholder(tf.float32,
shape=(None, image_height,
image_width, 3),
name='image')
phase_train_placeholder = tf.placeholder(tf.bool,
name='phase_train')
prelogits = network.infer(images_placeholder, feature_size)
if fc_bn:
prelogits = slim.batch_norm(
prelogits,
is_training=phase_train_placeholder,
epsilon=1e-5,
scale=True,
scope='softmax_bn')
embeddings = tf.identity(prelogits)
saver = tf.train.Saver(tf.global_variables(), max_to_keep=3)
saver.restore(sess, model)
for image_name in source_list:
if not full_path:
image_path = path.join(source, image_name)
else:
image_path = image_name
image_name = path.split(image_name)[1]
if not image_path.lower().endswith('.png') and not image_path.lower().endswith('.jpg') \
and not image_path.lower().endswith('.bmp'):
continue
dest_path = destination
if full_path:
sub_folder = path.basename(
path.normpath(path.split(image_path)[0]))
dest_path = path.join(destination, sub_folder)
if not path.exists(dest_path):
makedirs(dest_path)
features_name = path.join(dest_path, image_name[:-3] + 'npy')
images = utils.load_data([image_path], False, False,
image_height, image_width, prewhiten,
(image_height, image_width))
feed_dict = {
images_placeholder: images,
phase_train_placeholder: False
}
feats = sess.run(embeddings, feed_dict=feed_dict)
feats = utils.l2_normalize(feats)
np.save(features_name, feats)
def test(args):
with tf.Graph().as_default():
with tf.Session() as sess:
#saver = tf.train.Saver(tf.trainable_variables(), max_to_keep=3)
saver = tf.train.Saver(tf.global_variables())
saver.restore(sess, args.model)
# Read the file containing the pairs used for testing
pairs = lfw.read_pairs(os.path.expanduser(args.test_list_dir))
# Get the paths for the corresponding images
paths, actual_issame = lfw.get_paths(
os.path.expanduser(args.test_data_dir), pairs,
args.test_list_dir)
image_size = args.image_size
print('image size', image_size)
images_placeholder = tf.placeholder(tf.float32,
shape=(None, args.image_height,
args.image_width,
args.image_width),
name='image')
phase_train_placeholder = tf.placeholder(tf.bool,
name='phase_train')
#network definition.
prelogits1 = network.infer(images_placeholder, args.embedding_size)
if args.fc_bn:
print('do batch norm after network')
prelogits = slim.batch_norm(
prelogits1,
is_training=phase_train_placeholder,
epsilon=1e-5,
scale=True,
scope='softmax_bn')
#embeddings = tf.nn.l2_normalize(prelogits, 1, 1e-10, name='embeddings')
embeddings = tf.identity(prelogits)
embedding_size = embeddings.get_shape()[1]
# Run forward pass to calculate embeddings
print('Runnning forward pass on testing images')
batch_size = args.test_batch_size
nrof_images = len(paths)
nrof_batches = int(math.ceil(1.0 * nrof_images / batch_size))
emb_array = np.zeros((nrof_images, embedding_size))
for i in range(nrof_batches):
start_index = i * batch_size
print('handing {}/{}'.format(start_index, nrof_images))
end_index = min((i + 1) * batch_size, nrof_images)
paths_batch = paths[start_index:end_index]
images = utils.load_data(paths_batch, False, False, args.image_height,args.image_width,False,\
(args.image_height,args.image_width))
feed_dict = {
images_placeholder: images,
phase_train_placeholder: False
}
feats, a = sess.run([embeddings, prelogits],
feed_dict=feed_dict)
# do not know for sure whether we should turn this on? it depends.
feats = utils.l2_normalize(feats)
emb_array[start_index:end_index, :] = feats
tpr, fpr, accuracy, val, val_std, far = lfw.evaluate(
emb_array,
actual_issame,
0.001,
nrof_folds=args.test_nrof_folds)
print('Accuracy: %1.3f+-%1.3f' %
(np.mean(accuracy), np.std(accuracy)))
print('Validation rate: %2.5f+-%2.5f @ FAR=%2.5f' %
(val, val_std, far))
auc = metrics.auc(fpr, tpr)
print('Area Under Curve (AUC): %1.3f' % auc) #
eer = brentq(lambda x: 1. - x - interpolate.interp1d(fpr, tpr)(x),
0., 1.) #fill_value="extrapolate"
print('Equal Error Rate (EER): %1.3f' % eer)
tpr1, fpr1, accuracy1, val1, val_std1, far1 = lfw.evaluate(
emb_array,
actual_issame,
0.0001,
nrof_folds=args.test_nrof_folds)
print('Accuracy: %1.3f+-%1.3f' %
(np.mean(accuracy1), np.std(accuracy1)))
print('Validation rate: %2.5f+-%2.5f @ FAR=%2.5f' %
(val1, val_std1, far1))
auc = metrics.auc(fpr1, tpr1)
print('Area Under Curve (AUC): %1.3f' % auc) #
eer = brentq(lambda x: 1. - x - interpolate.interp1d(fpr1, tpr1)
(x), 0., 1.) #fill_value="extrapolate"
print('Equal Error Rate (EER): %1.3f' % eer)