def __init__(self, K=0.5, T=0.1):
super(LSMLoss, self).__init__()
self.cs = nn.CosineSimilarity(dim=1)
self.K = K
self.T = T
def evaluate_top_n(model, size=None):
print("%s: Performing Top-N evaluation" %
(time.strftime("%Y/%m/%d-%H:%M:%S")))
code_snippets_file = './data/parallel_bodies_n1000'
descriptions_file = './data/parallel_desc_n1000'
test_dataset = CodeDescDataset(code_snippets_file, descriptions_file, size)
tokenized_code_data, code_mask, tokenized_desc_data, desc_mask = tokenize_data(
test_dataset)
tokenized_code_data = tokenized_code_data.to(torch.int64).to(
utils.get_best_device())
tokenized_desc_data = tokenized_desc_data.to(torch.int64).to(
utils.get_best_device())
code_mask = code_mask.to(utils.get_best_device())
desc_mask = desc_mask.to(utils.get_best_device())
print('Tokenized code data', tokenized_code_data.shape)
print('Tokenized desc data', tokenized_desc_data.shape)
code_embedding_data, desc_embedding_data = model(tokenized_code_data,
code_mask,
tokenized_desc_data,
desc_mask)
print('Code embedding data', code_embedding_data.shape)
print('Desc embedding data', desc_embedding_data.shape)
with torch.no_grad():
cosine_similarity = nn.CosineSimilarity(dim=1, eps=1e-6)
results = {}
for rowid, desc_embedding in enumerate(desc_embedding_data):
# Calculate the cosine similarity between the code and desc embeddings
code_desc_similarity = cosine_similarity(
code_embedding_data[rowid].reshape((1, -1)),
desc_embedding.reshape((1, -1)))
other_code_embeddings = numpy.delete(code_embedding_data.cpu(),
rowid, 0)
tiled_desc = torch.Tensor(
numpy.tile(desc_embedding.cpu(),
(other_code_embeddings.shape[0], 1)))
# print('Other + tiled', other_code_embeddings.shape, tiled_desc.shape)
# Calculate the cosine similarity between the description vector and all the code snippets excepting the code that matches the desc
ress = torch.Tensor(
cosine_similarity(other_code_embeddings,
tiled_desc)).to(utils.get_best_device())
results[rowid] = len(ress[ress >= code_desc_similarity])
top_1 = get_top_n(1, results)
top_3 = get_top_n(3, results)
top_5 = get_top_n(5, results)
top_15 = get_top_n(15, results)
print('Top 1', top_1)
print('Top 3', top_3)
print('Top 5', top_5)
print('Top 15', top_15)
queries_ids, relevant_docs, queries_result_list =\
prepare_embeddings_for_topn_evaluation(code_embedding_data, desc_embedding_data)
metrics = compute_metrics(queries_result_list, queries_ids,
relevant_docs)
output_scores(metrics)
metrics.update({
'Top_1': top_1,
'Top_3': top_3,
'Top_5': top_5,
'Top_15': top_15
})
return metrics
def train(train_loader, model, optimizer, epoch):
criterion = nn.L1Loss()
batch_time = AverageMeter()
losses = AverageMeter()
model.train()
cos = nn.CosineSimilarity(dim=1, eps=0)
get_gradient = sobel.Sobel().cuda()
end = time.time()
for i, sample_batched in enumerate(train_loader):
image, depth = sample_batched['image'], sample_batched['depth']
depth = depth.cuda(async=True)
image = image.cuda()
image = torch.autograd.Variable(image)
depth = torch.autograd.Variable(depth)
ones = torch.ones(depth.size(0), 1, depth.size(2),
depth.size(3)).float().cuda()
ones = torch.autograd.Variable(ones)
optimizer.zero_grad()
output = model(image)
depth_grad = get_gradient(depth)
output_grad = get_gradient(output)
depth_grad_dx = depth_grad[:, 0, :, :].contiguous().view_as(depth)
depth_grad_dy = depth_grad[:, 1, :, :].contiguous().view_as(depth)
output_grad_dx = output_grad[:, 0, :, :].contiguous().view_as(depth)
output_grad_dy = output_grad[:, 1, :, :].contiguous().view_as(depth)
depth_normal = torch.cat((-depth_grad_dx, -depth_grad_dy, ones), 1)
output_normal = torch.cat((-output_grad_dx, -output_grad_dy, ones), 1)
# depth_normal = F.normalize(depth_normal, p=2, dim=1)
# output_normal = F.normalize(output_normal, p=2, dim=1)
loss_depth = torch.log(torch.abs(output - depth) + 0.5).mean()
loss_dx = torch.log(torch.abs(output_grad_dx - depth_grad_dx) +
0.5).mean()
loss_dy = torch.log(torch.abs(output_grad_dy - depth_grad_dy) +
0.5).mean()
loss_normal = torch.abs(1 - cos(output_normal, depth_normal)).mean()
loss = loss_depth + loss_normal + (loss_dx + loss_dy)
losses.update(loss.data[0], image.size(0))
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
batchSize = depth.size(0)
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.sum:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
loss=losses))
def train_maximize(origpassport, fakepassport, model, optimizer, criterion, trainloader, device, scheme):
model.train()
loss_meter = 0
signloss_meter = 0
maximizeloss_meter = 0
mseloss_meter = 0
csloss_meter = 0
acc_meter = 0
signacc_meter = 0
start_time = time.time()
mse_criterion = nn.MSELoss()
cs_criterion = nn.CosineSimilarity() #余弦相似性
for k, (d, t) in enumerate(trainloader):
d = d.to(device)
t = t.to(device)
optimizer.zero_grad()
if scheme == 1:
pred = model(d)
else:
pred = model(d, ind=1) #private graph
loss = criterion(pred, t)
signloss = torch.tensor(0.).to(device)
signacc = torch.tensor(0.).to(device)
count = 0
for m in model.modules():
if isinstance(m, SignLoss):
signloss += m.loss
signacc += m.acc
count += 1
maximizeloss = torch.tensor(0.).to(device)
mseloss = torch.tensor(0.).to(device)
csloss = torch.tensor(0.).to(device)
for l, r in zip(origpassport, fakepassport):
mse = mse_criterion(l, r)
cs = cs_criterion(l.view(1, -1), r.view(1, -1)).mean()
csloss += cs
mseloss += mse
maximizeloss += 1 / mse
(loss + maximizeloss).backward() #csloss do not backward
# (loss).backward() #only cross-entropy loss backward
torch.nn.utils.clip_grad_norm_(fakepassport, 2) #梯度裁剪
optimizer.step()
acc = (pred.max(dim=1)[1] == t).float().mean()
loss_meter += loss.item()
acc_meter += acc.item()
signloss_meter += signloss.item()
signacc_meter += signacc.item() / count
maximizeloss_meter += maximizeloss.item()
mseloss_meter += mseloss.item()
csloss_meter += csloss.item()
print(f'Batch [{k + 1}/{len(trainloader)}]: '
f'Loss: {loss_meter / (k + 1):.4f} '
f'Acc: {acc_meter / (k + 1):.4f} '
f'Sign Loss: {signloss_meter / (k + 1):.4f} '
f'Sign Acc: {signacc_meter / (k + 1):.4f} '
f'MSE Loss: {mseloss_meter / (k + 1):.4f} '
f'Maximize Dist: {maximizeloss_meter / (k + 1):.4f} '
f'CS: {csloss_meter / (k + 1):.4f} ({time.time() - start_time:.2f}s)',
end='\r')
print()
loss_meter /= len(trainloader)
acc_meter /= len(trainloader)
signloss_meter /= len(trainloader)
signacc_meter /= len(trainloader)
maximizeloss_meter /= len(trainloader)
mseloss_meter /= len(trainloader)
csloss_meter /= len(trainloader)
return {'loss': loss_meter,
'signloss': signloss_meter,
'acc': acc_meter,
'signacc': signacc_meter,
'maximizeloss': maximizeloss_meter,
'mseloss': mseloss_meter,
'csloss': csloss_meter,
'time': start_time - time.time()}
from models.mtcnn import MTCNN
from models.mobilefacenet import MobileFacenet
device = "cuda" if torch.cuda.is_available() else "cpu"
dt = datetime.now().isoformat(' ', 'seconds')
print("[INFO {}] Running on device: {}".format(dt, device))
# MTCNN model to face detection
model = MTCNN(device=device)
embeddings = MobileFacenet()
state_dict = torch.load("./weights/068.ckpt", map_location="cpu")
embeddings.load_state_dict(state_dict["net_state_dict"])
embeddings.eval()
embeddings.to(device)
cos = nn.CosineSimilarity(dim=0, eps=1e-6)
test_trainsforms = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
def main():
namebank = []
with open("weights/name_bank.txt") as f:
namebank = f.read().strip('\n').split('\n')
facebank = torch.load("weights/face_bank.pth", map_location="cpu")
vid = cv2.VideoCapture(0)
while True:
def predict(self, xs, ys=None, cands=None, cands_txt=None, obs=None):
"""Produce a prediction from our model.
Update the model using the targets if available, otherwise rank
candidates as well if they are available and param is set.
"""
self.start = time.time()
if xs is None:
return [{}]
is_training = ys is not None
if is_training: #
text_cand_inds, loss_dict = None, None
negs = self.get_negs(xs, ys)
if len(negs) > 0:
self.model.train()
self.zero_grad()
if self.opt['loss'] == 'cosine':
xe, ye = self.model(xs, obs[0]['mem'], ys, negs)
y = Variable(-torch.ones(xe.size(0)))
y[0]= 1
loss = self.criterion(xe, ye, y)
else:
x = self.model(xs, obs[0]['mem'], ys, negs)
y = Variable(torch.LongTensor([0]))
loss = self.criterion(x.unsqueeze(0), y)
loss.backward()
self.update_params()
rest = 0
if self.start2 != 99:
rest = self.start-self.start2
self.start2 = time.time()
if self.opt['loss'] == 'cosine':
pred = nn.CosineSimilarity().forward(xe,ye)
else:
pred = x
metrics = self.compute_metrics(loss.item(),
pred.squeeze(0), self.start2-self.start, rest)
return [{'metrics':metrics}]
else:
fixed = False
if hasattr(self, 'fixedCands') and self.fixedCands:
self.take_next_utt=True
self.twohoputt=True
self.tricks=True
else:
self.take_next_utt = False
self.twohoputt=False
self.tricks=False
if cands is None or cands[0] is None or self.take_next_utt:
# cannot predict without candidates.
if self.fixedCands or self.take_next_utt:
cands_txt2 = [self.fixedCands_txt2]
fixed = True
else:
return [{}]
# test set prediction uses candidates
self.model.eval()
if fixed:
if obs[0]['episode_done']:
self.cands_done = []
if xs is None:
xs = Variable(torch.LongTensor([self.parse('nothing')]))
xs = xs.clone()
if self.tricks:
vv=self.history['last_utterance']
if len(vv) == 0:
xsq = Variable(torch.LongTensor([self.parse('nothing')]))
else:
xsq = Variable(torch.LongTensor([vv]))
else:
xsq = xs
mems= obs[0]['mem']
if self.tricks:
mems = []
if self.fixedX is None:
xe, ye = self.model(xsq, mems, ys, self.fixedCands)
self.fixedX = ye
else:
# fixed cand embed vectors are cached, dont't recompute
blah = Variable(torch.LongTensor([1]))
xe, ye = self.model(xsq, mems, ys, [blah])
ye = self.fixedX
pred = nn.CosineSimilarity().forward(xe,ye)
origxe = xe
origpred = pred
val,ind=pred.sort(descending=True)
origind = ind
ypredorig = self.fixedCands_txt[ind[0].item()] # match
ypred = cands_txt2[0][ind[0].item()] # reply to match
if self.opt.get('kvmemnn_debug', False):
print("twohop-range:", self.opt.get('twohop_range', 100))
for i in range(10):
txt1= self.fixedCands_txt[ind[i].item()]
txt2= cands_txt2[0][ind[i].item()]
print(i, txt1,'\n ', txt2)
tc = [ypred]
if self.twohoputt:
# now we rerank original cands against this prediction
zq = []
z = []
ztxt = []
newwords = {}
r = self.opt.get('twohop_range', 100)
for i in range(r):
c = self.fixedCands2[ind[i].item()]
ctxt = self.fixedCands_txt2[ind[i].item()]
if i < 10:
zq.append(c)
z.append(c)
ztxt.append(ctxt)
for w in c[0]:
newwords[w.item()] = True
xs2 = torch.cat(zq, 1)
if ((self.interactiveMode and self.twohoputt)
or cands[0] is None):
# used for nextutt alg in demo mode, get 2nd hop
blah = Variable(torch.LongTensor([1]))
if self.tricks:
xe, ye = self.model(xs2, obs[0]['mem'], ys, z)
else:
xe, ye = self.model(xs2, obs[0]['mem'], ys, [blah])
ye = self.fixedX
blend = self.opt.get('twohop_blend', 0)
if blend > 0:
xe = (1-blend)*xe + blend*origxe
pred = nn.CosineSimilarity().forward(xe,ye)
for c in self.cands_done:
for i in range(len(ztxt)):
if ztxt[i] == c:
# interactive heuristic: don't repeat yourself
pred[i] = -1000
val,ind=pred.sort(descending=True)
# predict the highest scoring candidate, and return it.
#print(" [query: " + self.v2t(xsq) + "]")
ps = []
for c in obs[0]['mem']:
ps.append(self.v2t(c))
#print(" [persona: " + '|'.join(ps) + "]")
#print(" [1st hop qmatch: " + ypredorig + "]")
#print(" [1st hop nextut: " + ypred + "]")
if self.tricks:
ypred = ztxt[ind[0].item()] # match
self.cands_done.append(ypred)
else:
ypred = self.fixedCands_txt[ind[0].item()] # match
ypred2 = cands_txt2[0][ind[0].item()] # reply to match
self.cands_done.append(ind[0].item())
#print(" [2nd hop nextut: " + ypred2 + "]")
tc = [ypred]
self.history['labels'] = [ypred]
#print(" [final pred: " + ypred + "]")
ret = [{'text': ypred, 'text_candidates': tc }]
return ret
elif self.take_next_utt and not self.interactiveMode:
xe, ye = self.model(xs2, obs[0]['mem'], ys, cands[0])
pred = nn.CosineSimilarity().forward(xe, ye)
xe, ye = self.model(xs, obs[0]['mem'], ys, cands[0])
origpred = nn.CosineSimilarity().forward(xe,ye)
if 'alpha' not in self.opt:
alpha=0.1
else:
alpha=self.opt['alpha']
pred = alpha*pred + 1*origpred
val,ind=pred.sort(descending=True)
# predict the highest scoring candidate, and return it.
ypred = cands_txt[0][ind[0].item()] # match
tc = []
for i in range(len(ind)):
tc.append(cands_txt[0][ind[i].item()])
else:
if self.opt['loss'] == 'cosine':
xe, ye = self.model(xs, obs[0]['mem'], ys, cands[0])
pred = nn.CosineSimilarity().forward(xe,ye)
else:
x = self.model(xs, obs[0]['mem'], ys, cands[0])
pred = x #.squeeze()
val,ind=pred.sort(descending=True)
ypred = cands_txt[0][ind[0].item()] # match
tc = []
for i in range(min(100, ind.size(0))):
tc.append(cands_txt[0][ind[i].item()])
ret = [{'text': ypred, 'text_candidates': tc }]
return ret
return [{}] * xs.size(0)
def _prediction_loop(
self, dataloader: DataLoader, description: str, prediction_loss_only: Optional[bool] = None
) -> PredictionOutput:
"""
Prediction/evaluation loop, shared by `evaluate()` and `predict()`.
Works both with or without labels.
"""
prediction_loss_only = prediction_loss_only if prediction_loss_only is not None else self.prediction_loss_only
# multi-gpu eval
if self.args.n_gpu > 1 and not isinstance(self.model, torch.nn.DataParallel):
model_pretrained = torch.nn.DataParallel(self.model_pretrained)
model_finetuned = torch.nn.DataParallel(self.model_finetuned)
else:
model_pretrained = self.model_pretrained
model_finetuned = self.model_finetuned
model_pretrained.to(self.args.device)
model_finetuned.to(self.args.device)
logger.info("***** Running %s *****", description)
logger.info(" Num examples = %d", len(dataloader.dataset))
logger.info(" Batch size = %d", dataloader.batch_size)
eval_losses: List[float] = []
preds: np.ndarray = None
label_ids: np.ndarray = None
model_pretrained.eval()
model_finetuned.eval()
cos_sim = torch.zeros([12, 12], dtype=torch.float64)
cos_sim_fun = nn.CosineSimilarity(dim=-1, eps=1e-6)
count = 0
for inputs in tqdm(dataloader, desc=description):
has_labels = any(inputs.get(k) is not None for k in ["labels", "masked_lm_labels"])
count += 1
for k, v in inputs.items():
inputs[k] = v.to(self.args.device)
with torch.no_grad():
outputs_pretrained = model_pretrained(**inputs)
outputs_finetuned = model_finetuned(**inputs)
# Code for Analysis
for i in range(len(outputs_pretrained[2])):
# Loop over the layer
pretrained_flatten = torch.flatten(outputs_pretrained[2][i], start_dim=-2)
finetuned_flatten = torch.flatten(outputs_finetuned[2][i], start_dim=-2)
# Compute the cosine similarity and average over batches
cos_sim[i, :] += torch.mean(cos_sim_fun(pretrained_flatten, finetuned_flatten), dim=0)
if has_labels:
step_eval_loss, logits = outputs_finetuned[:2]
eval_losses += [step_eval_loss.mean().item()]
else:
logits = outputs_finetuned[0]
if not prediction_loss_only:
if preds is None:
preds = logits.detach().cpu().numpy()
else:
preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
if inputs.get("labels") is not None:
if label_ids is None:
label_ids = inputs["labels"].detach().cpu().numpy()
else:
label_ids = np.append(label_ids, inputs["labels"].detach().cpu().numpy(), axis=0)
### Plotting for fig 5
# Averaged over number of data and batch
cos_sim /= count
attention_type = self.args.attention_type
dataset = self.args.dataset_name
np.save('/mnt/c/Users/kaise/Desktop/researchData/' + dataset + '/' + attention_type + '_cos_sim.npy', cos_sim)
plt.figure()
plt.imshow(cos_sim.numpy(), vmin=0, vmax=1.0, cmap='Blues_r')
plt.colorbar()
plt.ylabel("Layer")
plt.xlabel("Head")
plt.title(attention_type + " cos similarity")
plt.savefig('/mnt/c/Users/kaise/Desktop/researchData/' + dataset + '/' + attention_type + '_cos_sim_' + dataset + '.png')
if self.compute_metrics is not None and preds is not None and label_ids is not None:
metrics = self.compute_metrics(EvalPrediction(predictions=preds, label_ids=label_ids))
else:
metrics = {}
if len(eval_losses) > 0:
metrics["loss"] = np.mean(eval_losses)
return PredictionOutput(predictions=preds, label_ids=label_ids, metrics=metrics)
random.seed(a=None)
font_path = '/Library/Fonts/NanumGothic.ttf'
fontprop = fm.FontProperties(fname=font_path, size=15)
#load object detection model
model_path = "./checkpoints/1020.pth"
device = torch.device('cpu')
model = get_hourglass['large_hourglass']
model.load_state_dict(torch.load(model_path, map_location=device))
model = model.eval()
#load recognition model
mat = databaseMat()
ids, embedding = mat.getMat()
img2vec = img2vec()
cossim = cos = nn.CosineSimilarity()
train_img_dir = os.listdir("./data/retail/images")
#images_path = "./data/retail/images/"+train_img_dir[random.randint(1,150)]
images_path = "./data/retail/images/"+train_img_dir[40]
test_images_path = "/Users/seungyoun/Downloads/test0.jpeg"
score_threshold = 0.5
start = time.time()
#load a image
img = cv2.imread(images_path)
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
height, width = img.shape[0], img.shape[1]
def __init__(self, mem_dim):
super(CosSimilarity, self).__init__()
self.cos = nn.CosineSimilarity(dim=mem_dim)
def cosine(source, target):
source, target = source.mean(), target.mean()
cos = nn.CosineSimilarity(dim=0)
loss = cos(source, target)
return loss.mean()
def forward_training(self,
video_fc_feat,
video_caption,
cap_length_list=None):
#pdb.set_trace()
batch_size = video_fc_feat.size(0)
batch_size_caption = video_caption.size(0)
video_state = self.init_hidden_new(batch_size)
caption_state = self.init_hidden_new(batch_size_caption)
caption_outputs = []
caption_time_step = video_caption.size(1)
for i in range(caption_time_step):
word = video_caption[:, i].clone()
#if video_caption[:, i].data.sum() == 0:
# break
#import ipdb
#pdb.set_trace()
caption_xt = self.word_embedding(word)
#caption_output, caption_state = self.lstm_caption.forward(caption_xt, caption_state)
caption_output, caption_state = self.lstm_caption.forward(
caption_xt, caption_state)
caption_outputs.append(caption_output)
caption_state = (caption_output, caption_state)
# caption_outputs: batch * caption_time_step * lstm_hidden_size
caption_outputs = torch.cat([_.unsqueeze(1) for _ in caption_outputs],
1).contiguous()
# LSTM encoding
video_outputs = []
for i in range(self.video_time_step):
video_xt = self.video_embedding(video_fc_feat[:, i, :])
video_output, video_state = self.lstm_video.forward(
video_xt, video_state)
video_outputs.append(video_output)
video_state = (video_output, video_state)
# video_outputs: batch * video_time_step * lstm_hidden_size
video_outputs = torch.cat([_.unsqueeze(1) for _ in video_outputs],
1).contiguous()
# soft attention for caption based on each video
output_list = list()
for i in range(self.video_time_step):
# part 1
video_outputs_linear = self.vid_linear(video_outputs[:, i, :])
video_outputs_linear_expand = video_outputs_linear.expand(
caption_outputs.size(1), video_outputs_linear.size(0),
video_outputs_linear.size(1)).transpose(0, 1)
# part 2
caption_outputs_flatten = caption_outputs.view(
-1, self.lstm_hidden_size)
caption_outputs_linear = self.sen_linear(caption_outputs_flatten)
caption_outputs_linear = caption_outputs_linear.view(
batch_size_caption, caption_outputs.size(1),
self.att_hidden_size)
# part 1 and part 2 attention
sig_probs = []
for cap_id in range(batch_size_caption):
#pdb.set_trace()
cap_length = max(cap_length_list[cap_id], 1)
caption_output_linear_cap_id = caption_outputs_linear[
cap_id, :cap_length, :]
video_outputs_linear_expand_clip = video_outputs_linear_expand[:, :
cap_length, :]
caption_outputs_linear_cap_id_exp = caption_output_linear_cap_id.expand_as(
video_outputs_linear_expand_clip)
video_caption = F.tanh(video_outputs_linear_expand_clip \
+ caption_outputs_linear_cap_id_exp)
video_caption_view = video_caption.contiguous().view(
-1, self.att_hidden_size)
video_caption_out = self.att_linear(video_caption_view)
video_caption_out_view = video_caption_out.view(-1, cap_length)
atten_weights = nn.Softmax(
dim=1)(video_caption_out_view).unsqueeze(2)
caption_output_cap_id = caption_outputs[cap_id, :cap_length, :]
caption_output_cap_id_exp = caption_output_cap_id.expand(batch_size,\
caption_output_cap_id.size(0), caption_output_cap_id.size(1))
atten_caption = torch.bmm(
caption_output_cap_id_exp.transpose(1, 2),
atten_weights).squeeze(2)
video_caption_hidden = torch.cat(
(atten_caption, video_outputs[:, i, :]), dim=1)
cos = nn.CosineSimilarity(dim=1, eps=1e-6)
cur_probs = cos(atten_caption,
video_outputs[:, i, :]).unsqueeze(1)
sig_probs.append(cur_probs)
sig_probs = torch.cat([_ for _ in sig_probs], 1).contiguous()
output_list.append(sig_probs)
simMM = torch.stack(output_list, dim=2).mean(2)
return simMM
def __init__(self, config):
super().__init__(config=config)
self.pdist = nn.CosineSimilarity()
def __init__(self):
super(TripletLoss, self).__init__()
self.cs = nn.CosineSimilarity(dim=1)
self.thr = 0.2
self.max_tplts = 2000
def __init__(self):
super(RAGTripletLoss, self).__init__()
self.cs = nn.CosineSimilarity(dim=1)
# self.cs = nn.CosineSimilarity(dim=1, eps=1e-3)
self.margin = 0.5
process:change into OrderDict,use load_state_dict -> nn.embedding
output:nn.Embedding
'''
num_embeddings, embedding_dim = weights_matrix.shape
from collections import OrderedDict
weight_ordered_dict = OrderedDict([('weight', torch.from_numpy(weights_matrix))])
emb_layer = nn.Embedding(num_embeddings, embedding_dim)
emb_layer.load_state_dict(weight_ordered_dict)
if non_trainable:
emb_layer.requires_grad = False
return emb_layer
model = LSTMLayer(hidden_dim, num_layers, output_size, vocabulary, glove)
model.to(device)
model.train()
cos = nn.CosineSimilarity(dim=1)
criterion = nn.MSELoss()
optimizer = optim.Adam(model.parameters(), lr=1e-3)
scheduler = StepLR(optimizer, step_size=1, gamma=0.7)
train_losses = []
dev_losses = []
train_accuracy_list = []
dev_accuracy_list = []
train_loss = 0
dev_loss = 0
test_loss = 0
train_accuracy = 0
dev_accuracy = 0
test_accuracy = 0
step = 0
def train_one_batch(inputs_batched,
labels_batched,
net,
seg_optimizer,
thr_optimizer,
list=None,
phase='original'):
criterion = nn.CrossEntropyLoss(ignore_index=255)
grad_criterion = TVLoss()
cos = nn.CosineSimilarity(dim=1, eps=1e-6)
###############################################
#### train segmenatation branch && backbone####
###############################################
feature_batched, predicts_batched = net(inputs_batched)
feature_batched = feature_batched.to(1)
predicts_batched = predicts_batched.to(1)
feature = torch.from_numpy(feature_batched.cpu().data.numpy()).to(1)
Softmax_predicts_batched = nn.Softmax(dim=1)(predicts_batched)
loss = criterion(predicts_batched, labels_batched)
seg_jac_loss = Overlap_loss(labels_batched,
Softmax_predicts_batched,
eps=1e-7)
sample_cosSim = Softmax_predicts_batched.mul(
make_one_hot(labels_batched)).sum(dim=1)
sample_cosSim_cpu = sample_cosSim.cpu().data.numpy()
if phase == 'mixup':
######### ManifoldMixuploss (MFMC loss)#########
_, _, MM_consistent_loss = ManifoldMixup_loss(
list['deep_feature1'], list['deep_feature2'], feature_batched,
list['feature_label1'], list['feature_label2'],
list['feature_mixup_label'], list['cosSim1'], list['cosSim2'],
list['random_lambda'])
MM_consistent_loss = cfg.weight_ManifoldMixuploss * MM_consistent_loss
######### Similoss (MCMC loss)#########
MSE = torch.nn.MSELoss(reduce=False, size_average=False)
Simi_consistent_loss = MSE(sample_cosSim, list['cosSimilarity'])
Simi_consistent_loss = torch.mean(Simi_consistent_loss,
dim=(1, 2)).sum()
Simi_consistent_loss = cfg.weight_Similoss * Simi_consistent_loss
# Simi_consistent_loss = cfg.weight_Similoss * MSE(sample_cosSim, list['cosSimilarity']).mean()
seg_loss = (loss + MM_consistent_loss + Simi_consistent_loss)
else:
seg_loss = (loss)
seg_optimizer.zero_grad()
seg_loss.backward()
seg_optimizer.step()
if phase == 'mixup':
#### train segmenatation branch ####
######### MixupFeatureUpdate #########
mixup_deep_feature = input_mixup(list['deep_feature1'].to(1),
list['deep_feature2'].to(1),
list['random_lambda'])
_, mixup_feature_predicts = net(inputs_batched,
feature_cat=mixup_deep_feature,
phase=phase)
mixup_deep_feature = torch.from_numpy(
mixup_deep_feature.cpu().data.numpy()).to(1)
mixup_feature_predicts = mixup_feature_predicts.to(1)
Softmax_predicts_batched = nn.Softmax(dim=1)(mixup_feature_predicts)
mixup_feature_loss = criterion(mixup_feature_predicts, labels_batched)
mixup_feature_jac_loss = Overlap_loss(labels_batched,
Softmax_predicts_batched,
eps=1e-7)
mixup_feature_seg_loss = cfg.weight_MixupFeatureUpdate * (
mixup_feature_loss)
seg_optimizer.zero_grad()
mixup_feature_seg_loss.backward()
seg_optimizer.step()
if phase == 'mixup':
return loss, seg_jac_loss, MM_consistent_loss, Simi_consistent_loss, mixup_feature_seg_loss
else:
return loss, seg_jac_loss, sample_cosSim_cpu, feature
def __init__(self, temp):
super().__init__()
self.temp = temp
self.cos = nn.CosineSimilarity(dim=-1)
def rank(data_source, label):
model.eval()
with torch.no_grad():
ndcg_acc = 0
ndcg_count = 0
ndcg_acc_1 = 0
ndcg_count_1 = 0
ndcg_acc_2 = 0
ndcg_count_2 = 0
total_loss = 0
ntokens = len(track_dic)
batch_size = data_source.size(1)
for i in range(0, data_source.size(0) - 1, seq_len):
data, targets = get_batch_past(data_source,
label,
i,
seq_len,
evaluation=True)
data = data.t()
targets = targets.t()
tracks_future, targets_future = get_batch_future(data_source,
label,
i,
seq_len,
evaluation=True)
tracks_future = tracks_future.t()
targets_future = targets_future.t()
#music_rnn; music_lstm
model.hidden = model.init_hidden()
rank_vec = model(data)[:, -1, :] # batch, ntokens [12, 1, 50704]
# advoid for loop
# advoid for loop
for j in range(batch_size):
track_f = tracks_future[j]
# remove padding elements
#track_f = track_f[track_f!=0]
cos = nn.CosineSimilarity(dim=1, eps=1e-6)
score = cos(rank_vec[j].unsqueeze(0), track_weight[track_f])
# get data frame without padding element
df_future = pd.DataFrame({
'track':
np.array(track_f),
'score':
np.array(score),
'skip_info':
np.array(targets_future[j][0:len(track_f)])
})
# remove padding elements
df_future = df_future.loc[df_future['track'] != 0]
# sort tracks_future according to score
df_future = df_future.sort_values(
by='score',
ascending=False) #0.8154440681444343 #0.8227163023038474
#df_future = df_future.sample(frac=1) # 0.8115378563756852 #0.7787248338261271
# NDCG
actual = dcg_score(df_future['skip_info'])
best = dcg_score(
df_future['skip_info'].sort_values(ascending=True))
if best: #best might be 0, while skip_info is 3,3,3,....
ndcg = actual / best
ndcg_acc = ndcg_acc + ndcg
else: # avoid nan
ndcg_acc = ndcg_acc + 1
ndcg_count = ndcg_count + 1
if (targets[j] == 0).sum() < x:
track_f = tracks_future[j]
cos = nn.CosineSimilarity(dim=1, eps=1e-6)
score_1 = cos(rank_vec[j].unsqueeze(0),
track_weight[track_f])
df_future_1 = pd.DataFrame({
'track':
np.array(track_f),
'score':
np.array(score_1),
'skip_info':
np.array(targets_future[j][0:len(track_f)])
})
df_future_1 = df_future_1.loc[df_future_1['track'] != 0]
df_future_1 = df_future_1.sort_values(
by='score', ascending=False
) #0.8154440681444343 #0.8227163023038474
# NDCG
actual_1 = dcg_score(df_future_1['skip_info'])
best_1 = dcg_score(
df_future_1['skip_info'].sort_values(ascending=True))
if best: #best might be 0, while skip_info is 3,3,3,....
ndcg_1 = actual_1 / best_1
ndcg_acc_1 = ndcg_acc_1 + ndcg_1
else: # avoid nan
ndcg_acc_1 = ndcg_acc_1 + 1
ndcg_count_1 = ndcg_count_1 + 1
else:
track_f = tracks_future[j]
cos = nn.CosineSimilarity(dim=1, eps=1e-6)
score_2 = cos(rank_vec[j].unsqueeze(0),
track_weight[track_f])
df_future_2 = pd.DataFrame({
'track':
np.array(track_f),
'score':
np.array(score_2),
'skip_info':
np.array(targets_future[j][0:len(track_f)])
})
df_future_2 = df_future_2.loc[df_future_2['track'] != 0]
df_future_2 = df_future_2.sort_values(
by='score', ascending=False
) #0.8154440681444343 #0.8227163023038474
# NDCG
actual_2 = dcg_score(df_future_2['skip_info'])
best_2 = dcg_score(
df_future_2['skip_info'].sort_values(ascending=True))
if best: #best might be 0, while skip_info is 3,3,3,....
ndcg_2 = actual_2 / best_2
ndcg_acc_2 = ndcg_acc_2 + ndcg_2
else: # avoid nan
ndcg_acc_2 = ndcg_acc_2 + 1
ndcg_count_2 = ndcg_count_2 + 1
ndcg_avg = ndcg_acc / ndcg_count
ndcg_avg_1 = ndcg_acc_1 / ndcg_count_1
ndcg_avg_2 = ndcg_acc_2 / ndcg_count_2
return ndcg_avg, ndcg_avg_1, ndcg_avg_2
class NTXEntCriterion(nn.Module):
"""Normalized, temperature-scaled cross-entropy criterion, as suggested in the SimCLR paper.
Parameters:
temperature (float, optional): temperature to scale the confidences. Defaults to 0.5.
"""
criterion = nn.CrossEntropyLoss(reduction="sum")
similarity = nn.CosineSimilarity(dim=2)
def __init__(self, temperature=0.5):
super(NTXEntCriterion, self).__init__()
self.temperature = temperature
self.batch_size = None
self.mask = None
def mask_correlated_samples(self, batch_size):
"""Masks examples in a batch and it's augmented pair for computing the valid summands for
the criterion.
Args:
batch_size (int): batch size of the individual batch (not including it's augmented pair)
Returns:
torch.Tensor: a mask (tensor of 0s and 1s), where 1s indicates a pair of examples in a
batch that will contribute to the overall batch loss
"""
mask = torch.ones((batch_size * 2, batch_size * 2), dtype=bool)
mask = mask.fill_diagonal_(0)
for i in range(batch_size):
mask[i, batch_size + i] = 0
mask[batch_size + i, i] = 0
return mask
def compute_similarities(self, z_i, z_j, temperature):
"""Computes the similarities between two projections `z_i` and `z_j`, scaling based on
`temperature`.
Args:
z_i (torch.Tensor): projection of a batch
z_j (torch.Tensor): projection of the augmented pair for the batch
temperature (float): temperature to scale the similarity by
Returns:
torch.Tensor: tensor of similarities for the positive and negative pairs
"""
batch_size = len(z_i)
mask = self.mask_correlated_samples(batch_size)
p1 = torch.cat((z_i, z_j), dim=0)
sim = self.similarity(p1.unsqueeze(1), p1.unsqueeze(0)) / temperature
sim_i_j = torch.diag(sim, batch_size)
sim_j_i = torch.diag(sim, -batch_size)
positive_samples = torch.cat((sim_i_j, sim_j_i),
dim=0).reshape(batch_size * 2, 1)
negative_samples = sim[mask].reshape(batch_size * 2, -1)
logits = torch.cat((positive_samples, negative_samples), dim=1)
return logits
def forward(self, z):
"""Computes the loss for a batch and its augmented pair.
Args:
z (torch.Tensor): tensor of a batch and it's augmented pair, concatenated
Returns:
torch.Tensor: loss for the given batch
"""
double_batch_size = len(z)
batch_size = double_batch_size // 2
z_i, z_j = z[:double_batch_size // 2], z[double_batch_size // 2:]
if self.batch_size is None or batch_size != self.batch_size:
self.batch_size = batch_size
self.mask = None
if self.mask is None:
self.mask = self.mask_correlated_samples(self.batch_size)
logits = self.compute_similarities(z_i, z_j, self.temperature)
labels = torch.zeros(self.batch_size * 2).long()
logits, labels = logits.to(z.device), labels.to(z.device)
loss = self.criterion(logits, labels)
loss /= 2 * self.batch_size
return loss
def predict(self, xs, ys=None, cands=None, cands_txt=None, obs=None):
"""
Produce a prediction from our model.
Update the model using the targets if available, otherwise rank candidates as
well if they are available and param is set.
"""
is_training = ys is not None
if is_training:
negs = self.get_negs(xs, ys)
if is_training and len(negs) > 0:
self.model.train()
self.optimizer.zero_grad()
if self.opt.get('input_dropout', 0) > 0:
xs, ys, negs = self.input_dropout(xs, ys, negs)
xe, ye = self.model(xs, ys, negs)
if self.debugMode:
# print example
print("inp: " + self.v2t(xs.squeeze()))
print("pos: " + self.v2t(ys.squeeze()))
for c in negs:
print("neg: " + self.v2t(c.squeeze()))
print("---")
y = -torch.ones(xe.size(0))
y[0] = 1
loss = self.criterion(xe, ye, y)
loss.backward()
self.optimizer.step()
pred = nn.CosineSimilarity().forward(xe, ye)
metrics = self.compute_metrics(loss.item(),
pred.data.squeeze())
return [{'metrics': metrics}]
else:
self.model.eval()
if cands is None or cands[0] is None:
# cannot predict without candidates.
if self.fixedCands:
cands = [self.fixedCands]
cands_txt = [self.fixedCands_txt]
else:
return [{'text': 'I dunno.'}]
# test set prediction uses fixed candidates
if self.fixedX is None:
xe, ye = self.model(xs, ys, self.fixedCands)
self.fixedX = ye
else:
# fixed candidate embed vectors are cached, dont't recompute
blah = torch.LongTensor([1])
xe, ye = self.model(xs, ys, [blah])
ye = self.fixedX
else:
# test set prediction uses candidates
xe, ye = self.model(xs, ys, cands[0])
pred = nn.CosineSimilarity().forward(xe, ye)
# This is somewhat costly which we could avoid if we do not evalute ranking.
# i.e. by only doing: val,ind = pred.max(0)
val, ind = pred.sort(descending=True)
# predict the highest scoring candidate, and return it.
ypred = cands_txt[0][ind.data[0]]
tc = []
for i in range(min(100, ind.size(0))):
tc.append(cands_txt[0][ind.data[i]])
ret = [{'text': ypred, 'text_candidates': tc}]
return ret
return [{'id': self.getID()}]
# create logger
# Define visulaize SummaryWriter instance
writer = SummaryWriter(logdir=args.check_path, filename_suffix='_first')
sys.stdout = NewLogger(osp.join(args.check_path, 'log.%s.txt' % time.strftime("%Y.%m.%d", time.localtime())))
kwargs = {'num_workers': args.nj, 'pin_memory': False} if args.cuda else {}
extract_kwargs = {'num_workers': args.nj, 'pin_memory': False} if args.cuda else {}
if not os.path.exists(args.check_path):
os.makedirs(args.check_path)
opt_kwargs = {'lr': args.lr, 'lr_decay': args.lr_decay, 'weight_decay': args.weight_decay, 'dampening': args.dampening,
'momentum': args.momentum}
l2_dist = nn.CosineSimilarity(dim=1, eps=1e-12) if args.cos_sim else nn.PairwiseDistance(p=2)
# pdb.set_trace()
if args.feat_format == 'kaldi':
file_loader = read_mat
elif args.feat_format == 'npy':
file_loader = np.load
torch.multiprocessing.set_sharing_strategy('file_system')
# train_dir = EgsDataset(dir=args.train_dir, feat_dim=args.feat_dim, loader=file_loader, transform=transform,
# batch_size=args.batch_size, random_chunk=args.random_chunk)
transform = ConcateNumInput_Test(num_frames=args.num_frames, remove_vad=args.remove_vad)
train_dir = ScriptTrainDataset(dir=args.train_dir, samples_per_speaker=args.input_per_spks,
loader=file_loader, transform=transform, num_valid=args.num_valid,
domain=False, rand_test=True)
def __init__(self, args, pretrained):
super(LATTE, self).__init__()
self.args = args
# 1. Character Embedding Layer
self.char_emb = nn.Embedding(args.char_vocab_size,
args.char_dim,
padding_idx=1)
nn.init.uniform_(self.char_emb.weight, -0.001, 0.001)
self.char_conv = nn.Sequential(
nn.Conv2d(1, args.char_channel_size,
(args.char_dim, args.char_channel_width)), nn.ReLU())
# 2. Word Embedding Layer
# initialize word embedding with GloVe
self.word_emb = nn.Embedding.from_pretrained(pretrained, freeze=True)
# highway network
assert self.args.hidden_size * 2 == (self.args.char_channel_size +
self.args.word_dim)
for i in range(2):
setattr(
self, 'highway_linear{}'.format(i),
nn.Sequential(
Linear(args.hidden_size * 2, args.hidden_size * 2),
nn.ReLU()))
setattr(
self, 'highway_gate{}'.format(i),
nn.Sequential(
Linear(args.hidden_size * 2, args.hidden_size * 2),
nn.Sigmoid()))
# 3. Contextual Embedding Layer
self.context_LSTM = LSTM(input_size=args.hidden_size * 2,
hidden_size=args.hidden_size,
bidirectional=True,
batch_first=True,
dropout=args.dropout)
# 4. Attention Flow Layer
self.att_weight_c = Linear(args.hidden_size * 2, 1)
self.att_weight_q = Linear(args.hidden_size * 2, 1)
self.att_weight_cq = Linear(args.hidden_size * 2, 1)
#-------------------------------------------------------------------------------------
# 5. feed_forward_network
self.fc1 = Linear(8800, 200)
self.fc2 = Linear(8800, 200) #11x800
self.relu = nn.ReLU()
# 6. Distribution Similarity
self.fc4 = Linear(2200, 2048) #11*200 up uc concate k=2048
self.fc3 = Linear(200, 2048) #
self.cosSi = nn.CosineSimilarity(dim=0, eps=1e-6) #dim 尺寸确定一下
# 7. Known Type Classifier
self.fc5 = Linear(2048, 3) ####???
self.fc6 = Linear(2048, 3)
# 8. ranking score layer
self.f_weight = Linear(200, 1)
self.g_weight = Linear(2048, 1)
def __init__(self, weighted=False):
super(CosineSimilarityLossWithMask, self).__init__()
self.CosineSimilarity = nn.CosineSimilarity(dim=1)
self.weighted = weighted
def cos(self, x, y):
"""
This returns the mean cosine similarity between two sets of vectors.
"""
c = nn.CosineSimilarity()
return c(x, y).mean()
def __init__(self, opt, t_lost):
self.opt = opt
self.t_lost = t_lost
self.trackers = []
self.track_idx = 0
self.cos = nn.CosineSimilarity()
cudnn.benchmark = True
# Define visulaize SummaryWriter instance
writer = SummaryWriter(logdir=args.ckp_dir, filename_suffix='soft')
kwargs = {'num_workers': 2, 'pin_memory': True} if args.cuda else {}
opt_kwargs = {
'lr': args.lr,
'lr_decay': args.lr_decay,
'weight_decay': args.weight_decay,
'dampening': args.dampening,
'momentum': args.momentum
}
if args.cos_sim:
l2_dist = nn.CosineSimilarity(dim=1, eps=1e-6)
else:
l2_dist = PairwiseDistance(2)
voxceleb, train_set, valid_set = wav_list_reader(args.dataroot, split=True)
# voxceleb2, voxceleb2_dev = voxceleb2_list_reader(args.dataroot)
# if args.makemfb:
# #pbar = tqdm(voxceleb)
# for datum in voxceleb:
# mk_MFB((args.dataroot +'/voxceleb1_wav/' + datum['filename']+'.wav'))
# print("Complete convert")
#
# if args.makespec:
# num_pro = 1.
# for datum in voxceleb:
sys.stdout = NewLogger(osp.join(args.check_path, 'log.txt'))
kwargs = {'num_workers': args.nj, 'pin_memory': True} if args.cuda else {}
if not os.path.exists(args.check_path):
os.makedirs(args.check_path)
opt_kwargs = {
'lr': args.lr,
'lr_decay': args.lr_decay,
'weight_decay': args.weight_decay,
'dampening': args.dampening,
'momentum': args.momentum
}
l2_dist = nn.CosineSimilarity(
dim=1, eps=1e-6) if args.cos_sim else PairwiseDistance(2)
transform = transforms.Compose([
concateinputfromMFB(num_frames=c.NUM_FRAMES_SPECT,
remove_vad=args.remove_vad),
# varLengthFeat(),
])
transform_T = transforms.Compose([
concateinputfromMFB(num_frames=c.NUM_FRAMES_SPECT,
input_per_file=args.test_input_per_file,
remove_vad=args.remove_vad),
# varLengthFeat(),
])
transform_V = transforms.Compose([
varLengthFeat(remove_vad=args.remove_vad),
])
def run_dialog(params,
dataset,
split,
aBot,
qBot=None,
beamSize=1):
assert aBot is not None or (qBot is not None and aBot is not None),\
"Must provide either an A-Bot alone or both \
Q-Bot and A-Bot when generating dialog"
rankMetrics, _ = rankQBot(qBot, dataset, 'val')
old_split = dataset.split
batchSize = dataset.batchSize
numRounds = dataset.numRounds
train_questions = set()
dataset.split = 'train'
dataloader = DataLoader(
dataset,
batch_size=batchSize,
shuffle=False,
num_workers=0,
collate_fn=dataset.collate_fn)
ind2word = dataset.ind2word
to_str_gt = lambda w: str(" ".join([ind2word[x] for x in filter(lambda x:\
x>0,w.data.cpu().numpy())])) #.encode('utf-8','ignore')
to_str_pred = lambda w, l: str(" ".join([ind2word[x] for x in list( filter(
lambda x:x>0,w.data.cpu().numpy()))][:l.data.cpu()[0]])) #.encode('utf-8','ignore')
for idx, batch in enumerate(dataloader):
# append all questions in train in a set to calculate downstream metrics
gtQuestions = Variable(batch['ques'], requires_grad=False)
gtQuesLens = Variable(batch['ques_len'], requires_grad=False)
if gtQuesLens.shape[0] < batchSize:
break
# iterate through the batch and add to dictionary
for j in range(batchSize):
for rnd in range(numRounds):
question_str = to_str_pred(gtQuestions[j,rnd,:], gtQuesLens[j,rnd])
train_questions.add(question_str[8:])
print("train questions len:", len(train_questions))
dataset.split = split
dataloader = DataLoader(
dataset,
batch_size=batchSize,
shuffle=False,
num_workers=0,
collate_fn=dataset.collate_fn)
text = {'data': []}
if '%s_img_fnames' % split not in dataset.data.keys():
print("[Error] Need coco directory and info as input " \
"to -cocoDir and -cocoInfo arguments for locating "\
"coco image files.")
print("Exiting dialogDump without saving files.")
return None
getImgFileName = lambda x: dataset.data['%s_img_fnames' % split][x]
getImgId = lambda x: int(getImgFileName(x)[:-4][-12:])
similarity_scores_mean = Variable(torch.zeros(numRounds))
norm_difference_scores_mean = Variable(torch.zeros(numRounds))
norm_scores_mean = Variable(torch.zeros(numRounds))
huber_scores_mean = Variable(torch.zeros(numRounds))
if params["useGPU"]:
similarity_scores_mean = similarity_scores_mean.cuda()
norm_difference_scores_mean = norm_difference_scores_mean.cuda()
norm_scores_mean = norm_scores_mean.cuda()
huber_scores_mean = huber_scores_mean.cuda()
tot_idx = 0
output_dialog = True
tot_examples = 0
unique_questions = 0
unique_questions_list = []
mutual_overlap_list = []
ent_1_list = []
ent_2_list = []
dist_1_list = []
dist_2_list = []
avg_precision_list = []
bleu_metric = 0
novel_questions = 0
oscillating_questions_cnt = 0
per_round_bleu = np.zeros(numRounds)
ent_1 = 0
ent_2 = 0
for idx, batch in enumerate(dataloader):
print("current batch:",idx)
if idx > 3:
output_dialog = False
tot_idx = tot_idx + 1
imgIds = [getImgId(x) for x in batch['index']]
dialog = [{'dialog': [], 'image_id': imgId} for imgId in imgIds]
if dataset.useGPU:
batch = {key: v.cuda() if hasattr(v, 'cuda')\
else v for key, v in batch.items()}
image = Variable(batch['img_feat'], volatile=True)
caption = Variable(batch['cap'], volatile=True)
# ignoring the last batch
if caption.size()[0] < batchSize:
break
captionLens = Variable(batch['cap_len'], volatile=True)
if qBot is None: # A-Bot alone needs ground truth dialog
gtQuestions = Variable(batch['ques'], volatile=True)
gtQuesLens = Variable(batch['ques_len'], volatile=True)
gtAnswers = Variable(batch['ans'], volatile=True)
gtAnsLens = Variable(batch['ans_len'], volatile=True)
if aBot:
aBot.eval(), aBot.reset()
aBot.observe(
-1, image=image, caption=caption, captionLens=captionLens)
if qBot:
qBot.eval(), qBot.reset()
qBot.observe(-1, caption=caption, captionLens=captionLens)
questions = []
for j in range(batchSize):
caption_str = to_str_gt(caption[j])[8:-6]
dialog[j]['caption'] = caption_str
past_dialog_hidden = None
cur_dialog_hidden = None
question_str_list = [[] for _ in range(batchSize)]
gt_questions_str = [[] for _ in range(batchSize)]
gtQuestions = Variable(batch['ques'], volatile=True)
gtQuesLens = Variable(batch['ques_len'], volatile=True)
gtAnswers = Variable(batch['ans'], volatile=True)
gtAnsLens = Variable(batch['ans_len'], volatile=True)
for round in range(numRounds):
if aBot is not None and qBot is None:
aBot.observe(
round,
ques=gtQuestions[:, round],
quesLens=gtQuesLens[:, round])
aBot.observe(
round,
ans=gtAnswers[:, round],
ansLens=gtAnsLens[:, round])
_ = aBot.forward()
answers, ansLens = aBot.forwardDecode(
inference='greedy', beamSize=beamSize)
elif aBot is not None and qBot is not None:
questions, quesLens = qBot.forwardDecode(
beamSize=beamSize, inference='greedy')
qBot.observe(round, ques=questions, quesLens=quesLens)
aBot.observe(round, ques=questions, quesLens=quesLens)
answers, ansLens = aBot.forwardDecode(
beamSize=beamSize, inference='greedy')
aBot.observe(round, ans=answers, ansLens=ansLens)
qBot.observe(round, ans=answers, ansLens=ansLens)
qBot.encoder()
cur_dialog_hidden = qBot.encoder.dialogHiddens[-1][0]
if round == 0:
past_dialog_hidden = qBot.encoder.dialogHiddens[-1][0]
cos = nn.CosineSimilarity(dim=1, eps=1e-6)
similarity_scores = cos(cur_dialog_hidden, past_dialog_hidden)
norm_difference_scores = torch.abs(torch.norm(cur_dialog_hidden, p=2, dim=1) - \
torch.norm(past_dialog_hidden,p=2,dim=1))
# calculate norm
norm_scores = torch.norm(cur_dialog_hidden, p=2, dim=1)
# calculate Huber Loss/ Difference at consecutive rounds with Huber Threshold = 0.1
threshold = 0.1
norm_differences = torch.abs(cur_dialog_hidden - past_dialog_hidden)
l2_mask = norm_differences <= threshold
norm_differences_new = 0.5 * norm_differences * norm_differences * (l2_mask == 1).float()
l1_mask = norm_differences > threshold
norm_differences_new = norm_differences_new + (((l1_mask == 1).float()) * (threshold *
(norm_differences - (0.5 * threshold))))
huber_scores = torch.sum(norm_differences_new, dim=1)
past_dialog_hidden = cur_dialog_hidden
similarity_scores_mean[round] = similarity_scores_mean[round] + torch.mean(similarity_scores)
norm_difference_scores_mean[round] = norm_difference_scores_mean[round] + torch.mean(norm_difference_scores)
norm_scores_mean[round] = norm_scores_mean[round] + torch.mean(norm_scores)
huber_scores_mean[round] = huber_scores_mean[round] + torch.mean(huber_scores)
for j in range(batchSize):
question_str = to_str_pred(questions[j], quesLens[j]) \
if qBot is not None else to_str_gt(gtQuestions[j])
gt_question_str = to_str_pred(gtQuestions[j,round,:], gtQuesLens[j,round])
gt_questions_str[j].append(gt_question_str[8:])
question_str_list[j].append(question_str[8:])
answer_str = to_str_pred(answers[j], ansLens[j])
if output_dialog:
if round == 0:
norm_score = float(norm_scores[j])
dialog[j]['dialog'].append({
"answer": answer_str[8:],
"question": question_str[8:] + ":" + "N:%.2f" % norm_score + " "
}) # "8:" for indexing out initial <START>
else:
similarity_score = float(similarity_scores[j])
norm_difference_score = float(norm_difference_scores[j])
norm_score = float(norm_scores[j])
huber_score = float(huber_scores[j])
dialog[j]['dialog'].append({
"answer": answer_str[8:],
"question": question_str[8:] + ":" + "C:%.2f" % similarity_score + ";" +
"NP:%.2f" % norm_difference_score + "H:%.2f" % huber_score + ";" +
"N:%.2f" % norm_score + " "
}) # "8:" for indexing out initial <START>
per_round_bleu_batch = np.zeros((numRounds, batchSize))
for j in range(batchSize):
# calculate bleu scores for each question str, with other questions as references to calculate
# mutual overlap
# also calculate round by round bleu score
unigrams = []
bigrams = []
avg_bleu_score = 0
for rnd in range(numRounds):
# Novel sentences metric
cur_ques = question_str_list[j][rnd]
gt_ques = gt_questions_str[j][rnd]
if cur_ques not in train_questions:
novel_questions += 1
# question oscillation metrics
if rnd >= 2:
if cur_ques == question_str_list[j][rnd-2]:
oscillating_questions_cnt += 1
# bleu/mutual overlap metric
references = []
for k in range(numRounds):
if rnd != k:
references.append(nltk.word_tokenize(question_str_list[j][k]))
avg_bleu_score += sentence_bleu(references,nltk.word_tokenize(cur_ques))
per_round_bleu_batch[rnd][j] = sentence_bleu([nltk.word_tokenize(gt_ques)],
nltk.word_tokenize(cur_ques))
unigrams.extend(list(ngrams(nltk.word_tokenize(cur_ques),1)))
bigrams.extend(list(ngrams(nltk.word_tokenize(cur_ques),2)))
avg_bleu_score /= float(numRounds)
mutual_overlap_list.append(avg_bleu_score)
bleu_metric += avg_bleu_score
tot_tokens = len(unigrams)
unigram_ctr = Counter(unigrams)
bigram_ctr = Counter(bigrams)
cur_ent_1 = get_entropy_ctr(unigram_ctr)
ent_1 += cur_ent_1
ent_1_list.append(cur_ent_1)
cur_ent_2 = get_entropy_ctr(bigram_ctr)
ent_2 += cur_ent_2
ent_2_list.append(cur_ent_2)
dist_1 = len(unigram_ctr.keys())/float(tot_tokens)
dist_2 = len(bigram_ctr.keys())/float(tot_tokens)
dist_1_list.append(dist_1)
dist_2_list.append(dist_2)
cur_unique_ques = len(set(question_str_list[j]))
unique_questions += cur_unique_ques
unique_questions_list.append(cur_unique_ques)
# dialog[j]['caption'] += ':' + str(cur_unique_ques)
tot_examples += batchSize
if output_dialog:
text['data'].extend(dialog)
per_round_bleu += np.sum(per_round_bleu_batch,axis=1)
avg_precision_list.extend(np.mean(per_round_bleu_batch,axis=0).tolist())
similarity_scores_mean = similarity_scores_mean * (1.0/tot_idx)
norm_difference_scores_mean = norm_difference_scores_mean * (1.0/tot_idx)
norm_scores_mean = norm_scores_mean *(1.0/tot_idx)
huber_scores_mean = huber_scores_mean *(1.0/tot_idx)
print("Mean Cos Similarity Scores:", similarity_scores_mean)
print("Mean Difference of Norms Scores:", norm_difference_scores_mean)
print("Mean Norm of Dialog State:", norm_scores_mean)
print("Mean Huber Loss(Norm of differences):", huber_scores_mean)
text['opts'] = {
'qbot': params['qstartFrom'],
'abot': params['startFrom'],
'backend': 'cudnn',
'beamLen': 20,
'beamSize': beamSize,
'decoder': params['decoder'],
'encoder': params['encoder'],
'gpuid': 0,
'imgNorm': params['imgNorm'],
'inputImg': params['inputImg'],
'inputJson': params['inputJson'],
'inputQues': params['inputQues'],
'loadPath': 'checkpoints/',
'maxThreads': 1,
'resultPath': 'dialog_output/results',
'sampleWords': 0,
'temperature': 1,
'useHistory': True,
'useIm': True,
}
unique_questions_arr = np.array(unique_questions_list)
# converting metrics to numpy arrays
similarity_scores_mean = similarity_scores_mean.cpu().data.numpy().tolist()
norm_difference_scores_mean = norm_difference_scores_mean.cpu().data.numpy().tolist()
norm_scores_mean = norm_scores_mean.cpu().data.numpy().tolist()
huber_scores_mean = huber_scores_mean.cpu().data.numpy().tolist()
bleu_metric /= float(tot_examples)
ent_1 /= float(tot_examples)
ent_2 /= float(tot_examples)
per_round_bleu = per_round_bleu / float(tot_examples)
print("tot unique questions: ", unique_questions)
print("tot examples: ", tot_examples)
print("avg unique questions per example: ", float(unique_questions) / tot_examples)
print("std unique questions per example: ", float(np.std(unique_questions_arr)))
print("Mutual Overlap (Bleu Metric): ", bleu_metric)
print("tot novel questions: ", novel_questions)
tot_questions = tot_examples * numRounds
print("tot questions: ", tot_questions)
print("avg novel questions: ", float(novel_questions)/float(tot_questions))
print("avg oscillating questions count", float(oscillating_questions_cnt)/tot_questions)
print("osciallation questions count", oscillating_questions_cnt)
dataset.split = old_split
ret_metrics = {}
ret_metrics["tot_unique_questions"] = unique_questions
ret_metrics["tot_examples"] = tot_examples
ret_metrics["mean_unique_questions"] = int((float(unique_questions) / tot_examples) * 100)/100.0
ret_metrics["std_unique_questions"] = int(float(np.std(unique_questions_arr)) * 100)/100.0
ret_metrics["similarity_scores_mean"] = similarity_scores_mean
ret_metrics["norm_difference_scores_mean"] = norm_difference_scores_mean
ret_metrics["norm_scores_mean"] = norm_scores_mean
ret_metrics["huber_scores_mean"] = huber_scores_mean
ret_metrics["mutual_overlap_score"] = bleu_metric
ret_metrics["tot_novel_questions"] = novel_questions
ret_metrics["avg_novel_questions"] = float(novel_questions)/float(tot_questions)
ret_metrics["tot_questions"] = tot_questions
ret_metrics['NLL'] = rankMetrics['logProbsMean']
ret_metrics["average_precision"] = np.mean(per_round_bleu)
ret_metrics["per_round_precision"] = per_round_bleu.tolist()
ret_metrics["ent_1"] = ent_1
ret_metrics["ent_2"] = ent_2
ret_metrics["dist_1"] = np.mean(dist_1_list)
ret_metrics["dist_2"] = np.mean(dist_2_list)
ret_metrics["average_precision_CI"] = (1.96 * np.std(avg_precision_list))/math.sqrt(len(avg_precision_list))
ret_metrics["ent_1_CI"] = (1.96 * np.std(ent_1_list))/math.sqrt(len(ent_1_list))
ret_metrics["ent_2_CI"] = (1.96 * np.std(ent_2_list))/math.sqrt(len(ent_2_list))
ret_metrics["unique_questions_CI"] = (1.96 * np.std(unique_questions_list))/math.sqrt(len(unique_questions_list))
ret_metrics["mutual_overlap_CI"] = (1.96 * np.std(mutual_overlap_list))/math.sqrt(len(mutual_overlap_list))
ret_metrics["dist_1_CI"] = (1.96 * np.std(dist_1_list))/math.sqrt(len(dist_1_list))
ret_metrics["dist_2_CI"] = (1.96 * np.std(dist_2_list))/math.sqrt(len(dist_2_list))
return text,ret_metrics
model.hidden = tuple(
map(lambda item: item.cuda(), model.hidden))
lstm_random_title = model(random_title_list[ridx])
#random body
model.hidden = model.init_hidden()
if use_gpu:
model.hidden = tuple(
map(lambda item: item.cuda(), model.hidden))
lstm_random_body = model(random_body_list[ridx])
lstm_random = (lstm_random_title + lstm_random_body) / 2.0
lstm_random_list.append(lstm_random)
#end for
cosine_similarity = nn.CosineSimilarity(dim=1, eps=1e-6)
score_pos = cosine_similarity(lstm_query, lstm_similar)
score_list = []
score_list.append(score_pos)
for ridx in range(len(lstm_random_list)):
score_neg = cosine_similarity(lstm_query, lstm_random_list[ridx])
score_list.append(score_neg)
X_scores = torch.stack(score_list, 1) #[batch_size, K=101]
y_targets = Variable(
torch.zeros(X_scores.size(0)).type(
torch.LongTensor)) #[batch_size]
if use_gpu:
y_targets = y_targets.cuda()
loss = criterion(X_scores, y_targets) #y_target=0
def __init__(self, gamma_3, embed_size=256):
super(Matching_Score_sent, self).__init__()
self.cos = nn.CosineSimilarity(dim=0)
self.gamma_3 = gamma_3
self.fc = nn.Linear(2048, embed_size)
