class SupervisedTest:
def __init__(self, window_size=3) -> None:
self.window_size = window_size
self.dataset = SupervisedDataset(mode='test',
window_size=self.window_size,
log_reg=False)
self.loader = DataLoader(self.dataset, batch_size=64, shuffle=False)
self.checkpoint_path = "./model_params/val/blstm_bs64_lr1e-3_ws40_hs128_nl2_dout50/val_lstm_epoch91_acc81.9593.pth"
#self.checkpoint_path = "./model_params/logistic_regression/val/val_logreg_epoch100_acc35.8229.pth"
self.checkpoint = torch.load(self.checkpoint_path)
self.model = LSTM(input_size=78,
hidden_size=128,
num_classes=170,
n_layers=2).to(device=torch.device('cuda:0'))
#self.model = LogisticRegression(num_keypoints=78, num_features=2, num_classes=170).to(device=torch.device('cuda:0'))
self.model.load_state_dict(self.checkpoint, strict=True)
self.model.eval()
#self.criterion = nn.BCEWithLogitsLoss() # Use this for Logistic Regression training
self.criterion = nn.BCELoss(
) # Use this for LSTM training (with Softmax)
#self.writer_text = SummaryWriter('./Tensorboard/test_text/')
#self.writer_avg_test_loss = SummaryWriter('./Tensorboard/test_loss/')
#self.writer_hparams = SummaryWriter('./Tensorboard/test_hparams/')
def _start_batch_test(self) -> None:
current_iter = 0
running_loss = 0.0
average_loss = 0.0
num_data = 0
running_correct_preds = 0
running_correct_classwise_preds = [0] * 170
running_false_classwise_preds = [0] * 170
running_all_classwise_gt_labels = [0] * 170
with torch.no_grad():
for batch_window, batch_label in self.loader:
current_iter += 1
outs = self.model(batch_window)
loss = self.criterion(outs, batch_label)
running_loss += loss.item()
average_loss = running_loss / current_iter
pred_confidence, pred_index = torch.max(outs, dim=1)
gt_confidence, gt_index = torch.max(batch_label, dim=1)
#batch_correct_preds = torch.eq(pred_index, gt_index).long().sum().item()
#batch_accuracy = (batch_correct_preds / batch_window.shape[0]) * 100
num_data += batch_window.shape[0]
batch_accuracy, batch_correct_preds, classwise_correct_preds, classwise_false_preds, classwise_gt_labels = self._calculate_batch_accuracy(
outs, batch_label)
running_correct_preds += batch_correct_preds
running_correct_classwise_preds = self._add_lists_elementwise(
running_correct_classwise_preds, classwise_correct_preds)
running_false_classwise_preds = self._add_lists_elementwise(
running_false_classwise_preds, classwise_false_preds)
running_all_classwise_gt_labels = self._add_lists_elementwise(
running_all_classwise_gt_labels, classwise_gt_labels)
if current_iter % 1 == 0:
print(
f"\nITER#{current_iter} BATCH TEST ACCURACY: {batch_accuracy:.4f}, RUNNING TEST LOSS: {loss.item():.8f}"
)
print(f"Predicted / GT index:\n{pred_index}\n{gt_index}\n")
#epoch_accuracy = (running_correct_preds / num_data) * 100
epoch_accuracy, classwise_accuracy = self._calculate_epoch_accuracy(
running_correct_preds, running_correct_classwise_preds,
running_all_classwise_gt_labels, num_data)
print(
f"\n\nTEST WINDOW-WISE ACCURACY: {epoch_accuracy:.4f}, AVERAGE TEST LOSS: {average_loss:.8f}\n\n"
)
correct_vid = 0
false_vid = 0
for i in range(len(running_correct_classwise_preds)):
print(
f"Person{i:03d} | Number of correct/all predictions: {running_correct_classwise_preds[i]:<3d}/{running_all_classwise_gt_labels[i]:<5d} | Accuracy: {classwise_accuracy[i]:.2f}%"
)
if (running_correct_classwise_preds[i] +
running_false_classwise_preds[i]) != 0:
if classwise_accuracy[i] >= 50:
correct_vid += 1
else:
false_vid += 1
videowise_accuracy = (correct_vid /
(correct_vid + false_vid)) * 100
print(
f"\n\nTEST VIDEO-WISE ACCURACY: {videowise_accuracy:.4f}%\n\n")
def start(self, mode="batch") -> None:
if mode == "batch":
self._start_batch_test()
def _calculate_batch_accuracy(self, predictions, annotations):
pred_confidence, pred_index = torch.max(predictions, dim=1)
gt_confidence, gt_index = torch.max(annotations, dim=1)
person1 = 1
if gt_index[0] == 0:
person1 += gt_index.shape[0]
batch_correct_preds = torch.eq(pred_index,
gt_index).long().sum().item()
batch_accuracy = (batch_correct_preds / predictions.shape[0]) * 100
# Calculating number of classwise correct/false predictions
classwise_correct_preds = torch.zeros(170).long()
classwise_false_preds = torch.zeros(170).long()
classwise_gt_labels = torch.zeros(170).long()
correct_preds_class = pred_index[torch.eq(pred_index, gt_index)].long()
false_preds_class = pred_index[torch.ne(pred_index, gt_index)].long()
for element in correct_preds_class:
classwise_correct_preds[element] += 1
for element in false_preds_class:
classwise_false_preds[element] += 1
for element in gt_index:
classwise_gt_labels[element] += 1
classwise_correct_preds = classwise_correct_preds.tolist()
classwise_false_preds = classwise_false_preds.tolist()
return batch_accuracy, batch_correct_preds, classwise_correct_preds, classwise_false_preds, classwise_gt_labels
def _add_lists_elementwise(self, list1, list2):
array1 = np.array(list1)
array2 = np.array(list2)
sum_list = (array1 + array2).tolist()
return sum_list
def _calculate_epoch_accuracy(self, running_correct_preds,
running_correct_classwise_preds,
running_all_classwise_gt_labels, num_data):
epoch_accuracy = (running_correct_preds / num_data) * 100
classwise_accuracy = [
0
] * 170 #(((np.array(running_correct_classwise_preds) / (running_correct_classwise_preds + running_false_classwise_preds))) * 100).tolist()
for i in range(len(running_correct_classwise_preds)):
if (running_all_classwise_gt_labels[i]) == 0:
classwise_accuracy[i] = 0
else:
classwise_accuracy[i] = (
running_correct_classwise_preds[i] /
running_all_classwise_gt_labels[i]) * 100
return epoch_accuracy, classwise_accuracy
n_samples_per_card = X_test[0].shape[0]
n_digits = 11
# Retrieves DeepSets model
deepsets = DeepSets(n_digits, embedding_dim, hidden_dim).to(device)
print("Loading DeepSets checkpoint!")
checkpoint = torch.load('model_deepsets.pth.tar')
deepsets.load_state_dict(checkpoint['state_dict'])
deepsets.eval()
# Retrieves LSTM model
lstm = LSTM(n_digits, embedding_dim, hidden_dim).to(device)
print("Loading LSTM checkpoint!")
checkpoint = torch.load('model_lstm.pth.tar')
lstm.load_state_dict(checkpoint['state_dict'])
lstm.eval()
# Dict to store the results
results = {'deepsets': {'acc': [], 'mae': []}, 'lstm': {'acc': [], 'mae': []}}
for i in range(len(cards)):
print(f"Cardinality: {cards[i]}, i: {i}")
y_pred_deepsets = list()
y_pred_lstm = list()
for j in range(0, n_samples_per_card, batch_size):
# Task 6
##################
x_test_batch = torch.tensor(X_test[i][j:j + batch_size]).to(
device, dtype=torch.long)
def main():
global args, best_auc
args = parser.parse_args()
cuda_available = torch.cuda.is_available()
print args
embedding_file = 'data/glove/glove.pruned.txt.gz'
embedding_iter = Embedding.iterator(embedding_file)
embed_size = 300
embedding = Embedding(embed_size, embedding_iter)
print 'Embeddings loaded.'
android_corpus_file = 'data/android/corpus.tsv.gz'
android_dataset = AndroidDataset(android_corpus_file)
android_corpus = android_dataset.get_corpus()
android_ids = embedding.corpus_to_ids(android_corpus)
print 'Got Android corpus ids.'
ubuntu_corpus_file = 'data/askubuntu/text_tokenized.txt.gz'
ubuntu_dataset = UbuntuDataset(ubuntu_corpus_file)
ubuntu_corpus = ubuntu_dataset.get_corpus()
ubuntu_ids = embedding.corpus_to_ids(ubuntu_corpus)
print 'Got AskUbuntu corpus ids.'
padding_id = embedding.vocab_ids['<padding>']
dev_pos_file = 'data/android/dev.pos.txt'
dev_neg_file = 'data/android/dev.neg.txt'
android_dev_data = android_dataset.read_annotations(
dev_pos_file, dev_neg_file)
android_dev_batches = batch_utils.generate_eval_batches(
android_ids, android_dev_data, padding_id)
assert args.model in ['lstm', 'cnn']
if os.path.isfile(args.load):
checkpoint = torch.load(args.load)
else:
print 'No checkpoint found here.'
return
if args.model == 'lstm':
encoder_src = LSTM(embed_size, args.hidden)
encoder_tgt = LSTM(embed_size, args.hidden)
else:
encoder_src = CNN(embed_size, args.hidden)
encoder_tgt = CNN(embed_size, args.hidden)
encoder_src.load_state_dict(checkpoint['state_dict'])
encoder_src.eval()
model_discrim = FFN(args.hidden)
print encoder_src
print encoder_tgt
print model_discrim
criterion = nn.CrossEntropyLoss()
if cuda_available:
criterion = criterion.cuda()
betas = (0.5, 0.999)
weight_decay = 1e-4
optimizer_tgt = torch.optim.Adam(encoder_tgt.parameters(),
lr=args.elr,
betas=betas,
weight_decay=weight_decay)
optimizer_discrim = torch.optim.Adam(model_discrim.parameters(),
lr=args.dlr,
betas=betas,
weight_decay=weight_decay)
for epoch in xrange(args.start_epoch, args.epochs):
train_batches = \
batch_utils.generate_classifier_train_batches(
ubuntu_ids, android_ids, args.batch_size,
args.batch_count, padding_id)
train_utils.train_adda(args, encoder_src, encoder_tgt, model_discrim,
embedding, optimizer_tgt, optimizer_discrim,
criterion, train_batches, padding_id, epoch)
auc = train_utils.evaluate_auc(args, encoder_tgt, embedding,
android_dev_batches, padding_id)
is_best = auc > best_auc
best_auc = max(auc, best_auc)
save(
args, {
'epoch': epoch + 1,
'arch': 'lstm',
'encoder_tgt_state_dict': encoder_tgt.state_dict(),
'discrim_state_dict': model_discrim.state_dict(),
'best_auc': best_auc,
}, is_best)
train_among_epochs = []
stop = False
for epoch in range(args.epochs):
start = time.time()
model.train()
train_loss = AverageMeter()
# Train for one epoch
for batch in range(n_train_batches):
output, loss = train(epoch, adj_train[batch], features_train[batch], y_train[batch])
train_loss.update(loss.data.item(), output.size(0))
# Evaluate on validation set
model.eval()
#for i in range(n_val_batches):
output, val_loss = test(adj_val[0], features_val[0], y_val[0])
val_loss = float(val_loss.detach().cpu().numpy())
# Print results
if(epoch%50==0):
#print("Epoch:", '%03d' % (epoch + 1), "train_loss=", "{:.5f}".format(train_loss.avg), "time=", "{:.5f}".format(time.time() - start))
print("Epoch:", '%03d' % (epoch + 1), "train_loss=", "{:.5f}".format(train_loss.avg),"val_loss=", "{:.5f}".format(val_loss), "time=", "{:.5f}".format(time.time() - start))
train_among_epochs.append(train_loss.avg)
val_among_epochs.append(val_loss)
#print(int(val_loss.detach().cpu().numpy()))
def main(load=False):
# Init hps
hps = init_hps()
criterion = nn.CrossEntropyLoss()
torch.manual_seed(0)
# Read file
if load:
print("Loading file", data_file, "for testing")
else:
print("Using file", data_file, "for training")
lines = utils.read_file(data_file)
global data_file_size
data_file_size = len(lines)
start = time.time()
unique_words, vocab_size, n = utils.create_unique_words(lines)
print("vocab_size", vocab_size)
print("Constructing unique words took:", (time.time() - start))
# Construct dataloader
dataset = utils.ReadLines(data_file)
print("data set length:", len(dataset))
train_set_len = int(len(dataset) * 0.6)
test_set_len = int(len(dataset) * 0.2)
validation_set_len = int(len(dataset) * 0.2)
while train_set_len + test_set_len + validation_set_len != len(dataset):
validation_set_len += 1
train_set, test_set, validation_set = torch.utils.data.random_split(
dataset, [train_set_len, test_set_len, validation_set_len])
train_loader = torch.utils.data.DataLoader(dataset=train_set,
batch_size=hps.batch_size,
num_workers=8,
shuffle=True,
collate_fn=collate_fn)
test_loader = torch.utils.data.DataLoader(dataset=test_set,
batch_size=hps.batch_size,
num_workers=8,
shuffle=True,
collate_fn=collate_fn)
validation_loader = torch.utils.data.DataLoader(dataset=validation_set,
batch_size=hps.batch_size,
num_workers=8,
shuffle=True,
collate_fn=collate_fn)
# Init model
if not load:
word_to_idx, idx_to_word = utils.build_index(unique_words)
mapper = SentenceMapper(lines, word_to_idx, idx_to_word)
vocab_info = {
'idx_to_word': idx_to_word,
'word_to_idx': word_to_idx,
'vocab_size': vocab_size
}
with open(
vocab_info_save_path(data_file_size, hps.lstm_h_dim,
hps.embedding_dim), 'wb') as f:
pickle.dump(vocab_info, f, protocol=pickle.HIGHEST_PROTOCOL)
embedding = fasttext.train_unsupervised(data_file,
model='cbow',
dim=hps.embedding_dim)
embedding.save_model(
embedding_model_save_path(data_file_size, hps.lstm_h_dim,
hps.embedding_dim))
print("Training...")
model = LSTM(hps, vocab_size)
train_model(hps, idx_to_word, model, train_loader, validation_loader,
mapper, embedding)
else:
with open(vocab_info_load_path, 'rb') as f:
vocab_info = pickle.load(f, encoding='utf-8')
idx_to_word = vocab_info['idx_to_word']
word_to_idx = vocab_info['word_to_idx']
vocab_size = vocab_info['vocab_size']
mapper = SentenceMapper(lines, word_to_idx, idx_to_word)
embedding = fasttext.load_model(
embedding_model_save_path(data_file_size, hps.lstm_h_dim,
hps.embedding_dim))
print("Loading model...")
model = LSTM(hps, vocab_size)
model = nn.DataParallel(model).to(device)
model.load_state_dict(torch.load(model_load_path, map_location=device))
model.to(device)
model.eval()
counter = 0
perplexities = []
for _, (data, N) in enumerate(test_loader):
padded_data = mapper.pad_sentences(data, N)
og_inputs, targets = utils.inputs_and_targets_from_sequences(
padded_data)
inputs = mapper.map_sentences_to_padded_embedding(
og_inputs,
embedding=embedding,
embedding_size=hps.embedding_dim,
N=N)
targets = mapper.map_words_to_indices(targets, N=N)
if cuda:
inputs = inputs.cuda()
targets = targets.cuda()
outputs = model(inputs)
loss = criterion(outputs.permute(0, 2, 1), targets)
perplexities.append(np.exp(loss.detach().cpu().numpy()))
topk = F.softmax(outputs, dim=2)[0, :, :]
topk = torch.topk(topk, 1, dim=1)[1].squeeze(1)
# print(topk.shape)
outputs = F.softmax(outputs, dim=2)[0, :, :].detach().cpu().numpy()
outs = []
idxs = np.array(list(range(vocab_size)))
for i in range(outputs.shape[0]):
outs.append(np.random.choice(idxs, p=np.array(outputs[i, :])))
output = torch.tensor(outs)
input_sequence = og_inputs[0, :]
predicted_sequence = [
idx_to_word[c] for c in topk.detach().cpu().numpy()
]
sampled_sequence = [
idx_to_word[c] for c in output.detach().cpu().numpy()
]
print('\nInput sequence')
print(input_sequence)
print('\nPredicted sequence:')
print(predicted_sequence)
print('\nSampled sequence:')
print(sampled_sequence)
prev_word = ""
for i in range(1, len(predicted_sequence)):
words = input_sequence[:i]
predicted_next_word = predicted_sequence[i - 1]
sampled_next_word = sampled_sequence[i - 1]
if sampled_next_word == '</s>' and (
prev_word == '</s>' or input_sequence[i] == '</s>'):
break
prev_word = sampled_next_word
print(
" ".join(list(words)),
"[" + predicted_next_word + "|" + sampled_next_word + "]")
print("Moving on to next prediction....\n")
print(perplexities)
mean_perplexity = np.mean(perplexities)
print(f'Perplexity: {mean_perplexity}')
with open(
perplexity_test_save_path(data_file_size, hps.lstm_h_dim,
hps.embedding_dim), 'a') as f:
f.write(str(mean_perplexity) + "\n")
return vocab_size, hps
