def distill_test(conf):
model.eval()
if conf['model_name'] in ['GCN', 'APPNP']:
logits = model(G.ndata['feat'])
elif conf['model_name'] in ['GAT', 'SGAT']:
logits, G.edata['a'] = model(G.ndata['feat'])
elif conf['model_name'] in ['GraphSAGE', 'SGC']:
logits = model(G, G.ndata['feat'])
elif conf['model_name'] == 'MoNet':
us, vs = G.edges(order='eid')
udeg, vdeg = 1 / torch.sqrt(G.in_degrees(us).float()), 1 / torch.sqrt(G.in_degrees(vs).float())
pseudo = torch.cat([udeg.unsqueeze(1), vdeg.unsqueeze(1)], dim=1)
logits = model(G.ndata['feat'], pseudo)
elif conf['model_name'] == 'GCNII':
logits = model(features, adj)
else:
raise ValueError(f'Undefined Model')
logp = F.log_softmax(logits, dim=1)
loss_test = F.nll_loss(logp[idx_test], labels[idx_test])
preds = torch.argmax(logp, dim=1).cpu().detach()
teacher_preds = torch.argmax(cas[-1], dim=1).cpu().detach()
acc_test = accuracy(logp[idx_test], labels[idx_test])
acc_teacher_test = accuracy(cas[-1][idx_test], labels[idx_test])
same_predict = np.count_nonzero(teacher_preds[idx_test] == preds[idx_test]) / len(idx_test)
acc_dis = np.abs(acc_teacher_test.item() - acc_test.item())
print("Test set results: loss= {:.4f} acc_test= {:.4f} acc_teacher_test= {:.4f} acc_dis={:.4f} same_predict= {:.4f}".format(
loss_test.item(), acc_test.item(), acc_teacher_test.item(), acc_dis, same_predict))
return acc_test, logp, same_predict
def distill_test(conf, model, G, labels_init, labels, idx_test, cas):
model.eval()
if conf['model_name'] in ['GCN', 'APPNP', 'LogReg', 'MLP']:
logits = model(G.ndata['feat'])
elif conf['model_name'] == 'GAT':
logits, G.edata['a'] = model(G.ndata['feat'])
elif conf['model_name'] == 'GraphSAGE':
logits = model(G, G.ndata['feat'])
elif conf['model_name'] == 'PLP':
logits, G.edata['a'], G.ndata['alpha'], G.ndata['el'], G.ndata['er'] = \
model(G.ndata['feat'], labels_init)
logp = F.log_softmax(logits, dim=1)
loss_test = F.nll_loss(logp[idx_test], labels[idx_test])
preds = torch.argmax(logp, dim=1).cpu().detach()
teacher_preds = torch.argmax(cas[-1], dim=1).cpu().detach()
acc_test = accuracy(logp[idx_test], labels[idx_test])
acc_teacher_test = accuracy(cas[-1][idx_test], labels[idx_test])
same_predict = np.count_nonzero(
teacher_preds[idx_test] == preds[idx_test]) / len(idx_test)
acc_dis = np.abs(acc_teacher_test.item() - acc_test.item())
print(
"Test set results: loss= {:.4f} acc_test= {:.4f} acc_teacher_test= {:.4f} acc_dis={:.4f} same_predict= {:.4f}"
.format(loss_test.item(), acc_test.item(), acc_teacher_test.item(),
acc_dis, same_predict))
return acc_test, logp, same_predict
def train(all_logits, dur, epoch):
t0 = time.time()
model.train()
optimizer.zero_grad()
if conf['model_name'] in ['GCN', 'APPNP']:
logits = model(G.ndata['feat'])
elif conf['model_name'] in ['GAT', 'SGAT']:
logits, _ = model(G.ndata['feat'])
elif conf['model_name'] in ['GraphSAGE', 'SGC']:
logits = model(G, G.ndata['feat'])
elif conf['model_name'] == 'MoNet':
us, vs = G.edges(order='eid')
udeg, vdeg = 1 / torch.sqrt(G.in_degrees(us).float()), 1 / torch.sqrt(
G.in_degrees(vs).float())
pseudo = torch.cat([udeg.unsqueeze(1), vdeg.unsqueeze(1)], dim=1)
logits = model(G.ndata['feat'], pseudo)
elif conf['model_name'] == 'GCNII':
logits = model(features, adj)
else:
raise ValueError(f'Undefined Model')
logp = F.log_softmax(logits, dim=1)
# we only compute loss for labeled nodes
loss = F.nll_loss(logp[idx_train], labels[idx_train])
acc_train = accuracy(logp[idx_train], labels[idx_train])
loss.backward()
optimizer.step()
dur.append(time.time() - t0)
model.eval()
if conf['model_name'] in ['GCN', 'APPNP']:
logits = model(G.ndata['feat'])
elif conf['model_name'] in ['GAT', 'SGAT']:
logits, _ = model(G.ndata['feat'])
elif conf['model_name'] in ['GraphSAGE', 'SGC']:
logits = model(G, G.ndata['feat'])
elif conf['model_name'] == 'MoNet':
us, vs = G.edges(order='eid')
udeg, vdeg = 1 / torch.sqrt(G.in_degrees(us).float()), 1 / torch.sqrt(
G.in_degrees(vs).float())
pseudo = torch.cat([udeg.unsqueeze(1), vdeg.unsqueeze(1)], dim=1)
logits = model(G.ndata['feat'], pseudo)
elif conf['model_name'] == 'GCNII':
logits = model(features, adj)
else:
raise ValueError(f'Undefined Model')
logp = F.log_softmax(logits, dim=1)
# we save the logits for visualization later
all_logits.append(logp.cpu().detach().numpy())
loss_val = F.nll_loss(logp[idx_val], labels[idx_val])
acc_val = accuracy(logp[idx_val], labels[idx_val])
acc_test = accuracy(logp[idx_test], labels[idx_test])
print(
'Epoch %d | Loss: %.4f | loss_val: %.4f | acc_train: %.4f | acc_val: %.4f | acc_test: %.4f | Time(s) %.4f'
% (epoch, loss.item(), loss_val.item(), acc_train.item(),
acc_val.item(), acc_test.item(), dur[-1]))
return acc_val, loss_val
def distill_train(all_logits, dur, epoch):
t0 = time.time()
model.train()
optimizer.zero_grad()
if conf['model_name'] in ['GCN', 'APPNP']:
logits = model(G.ndata['feat'])
elif conf['model_name'] in ['GAT', 'SGAT']:
logits, _ = model(G.ndata['feat'])
elif conf['model_name'] in ['GraphSAGE', 'SGC']:
logits = model(G, G.ndata['feat'])
elif conf['model_name'] == 'MoNet':
us, vs = G.edges(order='eid')
udeg, vdeg = 1 / torch.sqrt(G.in_degrees(us).float()), 1 / torch.sqrt(G.in_degrees(vs).float())
pseudo = torch.cat([udeg.unsqueeze(1), vdeg.unsqueeze(1)], dim=1)
logits = model(G.ndata['feat'], pseudo)
elif conf['model_name'] == 'GCNII':
logits = model(features, adj)
else:
raise ValueError(f'Undefined Model')
# print(G.ndata['alpha'])
logp = F.log_softmax(logits, dim=1)
# we only compute loss for labeled nodes
# loss = F.nll_loss(logp[idx_train], labels[idx_train])
if args.asstype == 0:
loss = F.nll_loss(logp[idx_train], labels[idx_train]) - 0.5 * F.kl_div(logp, cas[-1])
else:
loss = F.kl_div(logp, cas[-1], reduction='batchmean')
acc_train = accuracy(logp[idx_train], labels[idx_train])
loss.backward()
optimizer.step()
dur.append(time.time() - t0)
model.eval()
if conf['model_name'] in ['GCN', 'APPNP', 'LogReg', 'MLP']:
logits = model(G.ndata['feat'])
elif conf['model_name'] == 'GAT':
logits = model(G.ndata['feat'])[0]
elif conf['model_name'] == 'GraphSAGE':
logits = model(G, G.ndata['feat'])
elif conf['model_name'] == 'PLP':
logits = model(G.ndata['feat'], labels_init)[0]
logp = F.log_softmax(logits, dim=1)
all_logits.append(logp.cpu().detach().numpy())
# if conf['model_name'] == 'PLP':
# loss_val = my_loss(logp[idx_no_train], cas[-1][idx_no_train]) + F.nll_loss(logp[idx_val], labels[idx_val])
# else:
# loss_val = my_loss(logp, cas[-1])
loss_val = my_loss(logp[idx_val], cas[-1][idx_val])
# loss_val = loss
# loss_val = F.nll_loss(logp[idx_val], labels[idx_val])
acc_val = accuracy(logp[idx_val], labels[idx_val])
acc_test = accuracy(logp[idx_test], labels[idx_test])
print('Epoch %d | Loss: %.4f | loss_val: %.4f | acc_train: %.4f | acc_val: %.4f | acc_test: %.4f | Time(s) %.4f' % (
epoch, loss.item(), loss_val.item(), acc_train.item(), acc_val.item(), acc_test.item(), dur[-1]))
return acc_val, loss_val
def f(w):
"""Weights predictions and returns an accuracy score.
Args:
w (float): Array of weights.
Returns:
1 - accuracy.
"""
# Ensuring that the sum of weights is one and avoids division by zero
w = w / max(w.sum(), c.EPSILON)
# Gathering the maximum label identifier
max_label = np.max(labels)
# Creating an array to hold the weighted predictions
w_preds = np.zeros((preds.shape[0], max_label + 1))
# For every possible prediction
for i in range(preds.shape[0]):
# For every possible classifier
for j in range(preds.shape[1]):
# Sums the weighted classifier's prediction to its position in the final array
w_preds[i][preds[i][j]] += w[j]
# Gathers the most weighted prediction
hat_preds = np.argmax(w_preds, axis=1)
# Calculates the accuracy
acc = m.accuracy(hat_preds, labels)
return 1 - acc
def f(w):
"""Weights predictions and returns an accuracy score.
Args:
w (float): Array of weights.
Returns:
1 - accuracy.
"""
# Rounding entire weights
w = np.round(w)
# Gathering the maximum label identifier
max_label = np.max(labels)
# Creating an array to hold the weighted predictions
w_preds = np.zeros((preds.shape[0], max_label + 1))
# For every possible prediction
for i in range(preds.shape[0]):
# For every possible classifier
for j in range(preds.shape[1]):
# Sums the boolean classifier's prediction to its position in the final array
w_preds[i][preds[i][j]] += w[j]
# Gathers the most weighted prediction
hat_preds = np.argmax(w_preds, axis=1)
# Calculates the accuracy
acc = m.accuracy(hat_preds, labels)
return 1 - acc
def evaluate(weights, preds, labels):
"""Evaluates an ensemble based on optimized weights and classifiers' predictions.
Args:
weights (np.array): Array of weights (n_classifiers, 1).
preds (np.array): Array of predictions of shape (n_samples, n_classifiers).
labels (np.array): Array of ground truth labels of shape (n_samples, 1).
Returns:
An accuracy score.
"""
# Gathering the maximum label identifier
max_label = np.max(labels)
# Creating an array to hold the weighted predictions
w_preds = np.zeros((preds.shape[0], max_label + 1))
# For every possible prediction
for i in range(preds.shape[0]):
# For every possible classifier
for j in range(preds.shape[1]):
# Sums the weighted classifier's prediction to its position in the final array
w_preds[i][preds[i][j]] += weights[j]
# Gathers the most weighted prediction
hat_preds = np.argmax(w_preds, axis=1)
# Calculates the accuracy
acc = m.accuracy(hat_preds, labels)
return acc
def train_model(model,
dataloader,
loss_fn,
optimizer,
epoch,
is_lstm,
use_cuda=False,
verbose=False):
# set model to train mode
model.train()
top1 = AverageMeter()
total_loss = 0
# loop through data batches
count = 0
for batch_idx, (X, y) in enumerate(tqdm(dataloader)):
batch_size = -1
# Utilize GPU if enabled
if use_cuda:
if is_lstm:
X['X'] = X['X'].cuda()
else:
X = X.cuda()
y = y.cuda(async=True)
if is_lstm:
batch_size = X['X'].size(0)
else:
batch_size = X.size(0)
# Compute loss
predictions = model(X)
count += predictions.shape[0]
loss = loss_fn(predictions, y)
total_loss += loss.item()
if verbose:
logging.debug('mini-batch loss: {}'.format(loss))
logging.debug('y: {}'.format(y))
# Compute running accuracy
acc1 = accuracy(predictions.data, y, (1, ))
top1.update(acc1[0], batch_size)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if verbose:
print('Progress [{0}/{1} ({2:.0f}%)]\tLoss:{3}'.format(
count, len(dataloader.dataset),
100. * batch_idx / len(dataloader), loss.item()))
total_loss /= count
train_acc = top1.avg
logging.info(
'Train Epoch: {} \tLoss: {:.6f} \t Training Acc: {:.2f}'.format(
epoch, total_loss, train_acc))
return total_loss, train_acc
def call(self, inputs: list, training: bool = True) -> \
Tuple[tf.Tensor, tf.Tensor]:
"""
Forward propagation
:param inputs: the information passed to next layers
:param training: whether in the training mode
"""
support, r_support, features, r_feature, label, idx_mask = inputs
# forward propagation
h_r = self.r_agg_layer((support, features))
h_u, h_i = self.iu_agg_layer((support, features))
p_e = self.r_gcn_layer((r_feature, r_support), training=True)
concat_vecs = [h_r, h_u, h_i, p_e]
gas_out = self.concat_layer((support, concat_vecs))
# get masked data
masked_data = tf.gather(gas_out, idx_mask)
masked_label = tf.gather(label, idx_mask)
# calculation loss and accuracy()
logits = tf.nn.softmax(tf.matmul(masked_data, self.u))
loss = -tf.reduce_sum(tf.math.log(tf.nn.sigmoid(
masked_label * logits)))
acc = accuracy(logits, masked_label)
return loss, acc
def nearest_neighbour(train_data, train_labels, test_data, test_labels):
print(f'{NearestNeighbour.__name__}:')
# Create and train model
model = NearestNeighbour(5, dist=manhattan)
model.train(train_data, train_labels)
# Predict 2000 validation set samples and calculate accuracy
test_data_2k = test_data[:len(test_labels)]
test_pred = model.predict(test_data_2k)
# Print metrics
print('\nTest Accuracy: {:.02f}%\n'.format(
100 * accuracy(test_pred, test_labels)))
mat, classes = confusion_matrix(test_pred, test_labels)
print('Precision:\n{}\n'.format(
np.round(precision(test_pred, test_labels), 2)))
print('Recall:\n{}\n'.format(np.round(recall(test_pred, test_labels), 2)))
print('F1:\n{}\n'.format(np.round(f1_score(test_pred, test_labels), 2)))
print('Confusion Matrix:')
print(mat)
# Predict 10000 test set samples and save predictions
print('Predicting 10k samples...')
test_pred = model.predict(test_data)
save_predictions(nearest_neighbour.__name__, test_pred)
print('Saved 10k predictions.\n')
def linear_svm(train_data, train_labels, test_data, test_labels):
print(f'{LinearSVM.__name__}:')
# Create and train model
lsvm_model = LinearSVM(alpha=0.01, features=180)
model = OneVersusRest(lsvm_model)
model.train(train_data, train_labels)
# Predict 2000 validation set samples and calculate accuracy
test_data_2k = test_data[:len(test_labels)]
test_pred = model.predict(test_data_2k)
# Print metrics
print('\nTest Accuracy: {:.02f}%\n'.format(
100 * accuracy(test_pred, test_labels)))
mat, classes = confusion_matrix(test_pred, test_labels)
print('Precision:\n{}\n'.format(
np.round(precision(test_pred, test_labels), 2)))
print('Recall:\n{}\n'.format(np.round(recall(test_pred, test_labels), 2)))
print('F1:\n{}\n'.format(np.round(f1_score(test_pred, test_labels), 2)))
print('Confusion Matrix:')
print(mat)
# Predict 10000 test set samples and save predictions
print('Predicting 10k samples...')
test_pred = model.predict(test_data)
save_predictions(linear_svm.__name__, test_pred)
print('Saved 10k predictions.\n')
def evaluate(dataset, model, target_size, return_pred=False, loss_function=None):
results = []
eval_loader = data_reader.data_loader(dataset, config.BATCH_SIZE, shuffle=False)
n_batches = int(np.ceil(len(dataset)/config.BATCH_SIZE))
pbar = tqdm(range(n_batches))
model.eval()
total_loss = 0.
for i in pbar:
x, x_rc, md, y, x_len = next(iter(eval_loader))
score = model(x, x_rc, md)
loss = 0.
if loss_function is not None:
loss = loss_function(score, y)
total_loss += loss.mean().item()
probs = torch.nn.Softmax(dim=1)(score)
results.append(probs.detach().cpu().numpy())
model.train()
results = np.vstack(results)
if loss_function is not None:
print('Val Loss: {:.5f}'.format(total_loss/n_batches))
if return_pred:
return results
y_pred = np.argmax(results, axis=1)
acc = accuracy(dataset['y'], y_pred)
top10 = top10_accuracy_scorer(np.array(dataset['y']), results, target_size=target_size)
f1 = f1_score(dataset['y'], y_pred)
# print(acc, top10, f1)
return acc, top10, f1
def main_test(args):
voc, char2id, id2char = get_vocabulary(voc_type=args.voc_type)
input_images = tf.placeholder(dtype=tf.float32, shape=[1, args.height, None, 3], name="input_images")
input_images_width = tf.placeholder(dtype=tf.float32, shape=[1], name="input_images_width")
input_labels = tf.placeholder(dtype=tf.int32, shape=[1, args.max_len], name="input_labels")
sar_model = SARModel(num_classes=len(voc),
encoder_dim=args.encoder_sdim,
encoder_layer=args.encoder_layers,
decoder_dim=args.decoder_sdim,
decoder_layer=args.decoder_layers,
decoder_embed_dim=args.decoder_edim,
seq_len=args.max_len,
is_training=False)
model_infer, attention_weights, pred = sar_model(input_images, input_labels, input_images_width, batch_size=1, reuse=False)
global_step = tf.get_variable('global_step', [], initializer=tf.constant_initializer(0), trainable=False, dtype=tf.int32)
variable_averages = tf.train.ExponentialMovingAverage(0.997, global_step)
saver = tf.train.Saver(variable_averages.variables_to_restore())
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
ckpt_state = tf.train.get_checkpoint_state(args.checkpoints)
model_path = os.path.join(args.checkpoints, os.path.basename(ckpt_state.model_checkpoint_path))
print('Restore from {}'.format(model_path))
saver.restore(sess, model_path)
images_path, labels = get_data(args)
predicts = []
for img_path, label in zip(images_path, labels):
try:
img = cv2.imread(img_path)
except Exception as e:
print("{} error: {}".format(img_path, e))
continue
img, la, width = data_preprocess(img, label, char2id, args)
pred_value, attention_weights_value = sess.run([pred, attention_weights], feed_dict={input_images: [img],
input_labels: [la],
input_images_width: [width]})
pred_value_str = idx2label(pred_value, id2char, char2id)[0]
print("predict: {} label: {}".format(pred_value_str, label))
predicts.append(pred_value_str)
pred_value_str += '$'
if args.vis_dir != None and args.vis_dir != "":
os.makedirs(args.vis_dir, exist_ok=True)
assert len(img.shape) == 3
att_maps = attention_weights_value.reshape([-1, attention_weights_value.shape[2], attention_weights_value.shape[3], 1]) # T * H * W * 1
for i, att_map in enumerate(att_maps):
if i >= len(pred_value_str):
break
att_map = cv2.resize(att_map, (img.shape[1], img.shape[0]))
_att_map = np.zeros(dtype=np.uint8, shape=[img.shape[0], img.shape[1], 3])
_att_map[:, :, -1] = (att_map * 255).astype(np.uint8)
show_attention = cv2.addWeighted(img, 0.5, _att_map, 2, 0)
cv2.imwrite(os.path.join(args.vis_dir, os.path.basename(img_path).split('.')[0] + "_" + str(i) + "_" + pred_value_str[i] + ".jpg"), show_attention)
acc_rate = accuracy(predicts, labels)
print("Done, Accuracy: {}".format(acc_rate))
def call(self, inputs):
""":param inputs include support, x, label, mask
support means a list of the sparse adjacency Tensor
x means feature
label means label tensor
mask means a list of mask tensors to obtain the train data
"""
supports, x, label, idx_mask = inputs
# forward propagation
outputs = [self.input_layer((x, supports, self.h_0))]
for layer in self.layers_:
hidden = layer((x, supports, outputs[-1]))
outputs.append(hidden)
gem_out = outputs[-1]
# get masked data
masked_data = tf.gather(gem_out, idx_mask)
masked_label = tf.gather(label, idx_mask)
# Eq. (7) in paper
logits = tf.nn.softmax(tf.matmul(masked_data, self.u))
loss = -tf.reduce_sum(tf.math.log(tf.nn.sigmoid(
masked_label * logits)))
acc = accuracy(logits, masked_label)
return loss, acc
def logistic_regression(train_data, train_labels, test_data, test_labels):
print(f'{LogisticRegression.__name__}:')
# Create and train model
lr_model = LogisticRegression(train_data.shape[1], eta=0.001, epochs=50)
model = OneVersusRest(lr_model)
model.train(train_data, train_labels)
# Predict 2000 validation set samples and calculate accuracy
test_data_2k = test_data[:len(test_labels)]
test_pred = model.predict(test_data_2k)
# Print metrics
print('\nTest Accuracy: {:.02f}%\n'.format(
100 * accuracy(test_pred, test_labels)))
mat, classes = confusion_matrix(test_pred, test_labels)
print('Precision:\n{}\n'.format(
np.round(precision(test_pred, test_labels), 2)))
print('Recall:\n{}\n'.format(np.round(recall(test_pred, test_labels), 2)))
print('F1:\n{}\n'.format(np.round(f1_score(test_pred, test_labels), 2)))
print('Confusion Matrix:')
print(mat)
# Predict 10000 test set samples and save predictions
print('Predicting 10k samples...')
test_pred = model.predict(test_data)
save_predictions(logistic_regression.__name__, test_pred)
print('Saved 10k predictions.\n')
def calc_metric(pred, true):
tmp_jac = jaccard(pred, true)
tmp_ham = hamming(pred, true)
tmp_prec = precision(pred, true)
tmp_rec = recall(pred, true)
tmp_f1 = f1_score(pred, true)
tmp_acc = accuracy(pred, true)
return tmp_jac, tmp_ham, tmp_prec, tmp_rec, tmp_f1, tmp_acc
def inference(sampler, dataloader, args):
output_inf_file = open(args.preds_dir + '.inf', mode='w', encoding='utf8')
greedy_stats = {}
greedy_accuracy = 0
for batch in dataloader:
sampler.set_batch(batch)
src = sampler.get_src_sentence(0)
trg = sampler.get_trg_sentence(0)
src_gender = sampler.get_src_gender(0)
trg_gender = sampler.get_trg_gender(0)
translated = sampler.greedy_decode(sentence=src, trg_gender=trg_gender)
greedy_accuracy += accuracy(trg=trg, pred=translated)
correct = 'CORRECT!' if trg == translated else 'INCORRECT!'
if translated == trg:
greedy_stats[(src_gender, trg_gender, 'correct')] = 1 + greedy_stats.get((src_gender, trg_gender, 'correct'), 0)
else:
greedy_stats[(src_gender, trg_gender, 'incorrect')] = 1 + greedy_stats.get((src_gender, trg_gender, 'incorrect'), 0)
output_inf_file.write(translated)
output_inf_file.write('\n')
logger.info(f'src:\t\t\t{src}')
logger.info(f'trg:\t\t\t{trg}')
logger.info(f'greedy:\t\t\t{translated}')
logger.info(f'src gender:\t\t{src_gender}')
logger.info(f'trg gender:\t\t{trg_gender}')
logger.info(f'res:\t\t\t{correct}')
logger.info('\n\n')
greedy_accuracy /= len(dataloader)
output_inf_file.close()
logger.info('*******STATS*******')
total_examples = sum([greedy_stats[x] for x in greedy_stats])
logger.info(f'TOTAL EXAMPLES: {total_examples}')
logger.info('\n')
correct_greedy = {(x[0], x[1]): greedy_stats[x] for x in greedy_stats if x[2] == 'correct'}
incorrect_greedy = {(x[0], x[1]): greedy_stats[x] for x in greedy_stats if x[2] == 'incorrect'}
total_correct_greedy = sum([v for k,v in correct_greedy.items()])
total_incorrect_greedy = sum([v for k, v in incorrect_greedy.items()])
logger.info('Results using greedy decoding:')
for x in correct_greedy:
logger.info(f'{x[0]}->{x[1]}')
logger.info(f'\tCorrect: {correct_greedy.get(x, 0)}\tIncorrect: {incorrect_greedy.get(x, 0)}')
logger.info(f'--------------------------------')
logger.info(f'Total Correct: {total_correct_greedy}\tTotal Incorrect: {total_incorrect_greedy}')
logger.info(f'Accuracy:\t{greedy_accuracy}')
def card(corpus):
global vectorizer
jac_list = []
ham_list = []
prec_list = []
rec_list = []
f1_list = []
acc_list = []
id2term = {}
for k, v in vectorizer.vocabulary_.items():
id2term[v] = k
for idx, item in enumerate(corpus):
docstring = vectorizer.transform([item[-1]])[0]
true_types = item[3]
terms = [id2term[i] for i in docstring.indices]
pred_types = []
# try hint terms of each basic type whether match a type
# term pattern `\w+(\_)?name|\w+(\_)?method`
# is more likely to be a `str` type
for basic_type in basic_types:
h_terms = hint_terms[basic_type]
if set(terms) & set(h_terms):
pred_types.append('List' if basic_type ==
'Tuple' else basic_type)
elif basic_type == 'str':
for term in terms:
if type_to_regexp[basic_type].match(term):
pred_types.append(basic_type)
break
# if type `Dict` and `str` in pred_types at same time
# remove `str` as type hardly never occur `Dict` and `str` at same time
if 'Dict' in pred_types and 'str' in pred_types:
pred_types.remove('str')
# type `Type` often occur independently
if 'Type' in pred_types and len(pred_types) > 1:
pred_types.remove('Type')
# if pred_types is empty
# then find whether exists a term whose part of speech(pos) is NNS or NNPS
# if found, the pos of this variable is likely to be `List`
if not pred_types:
pos_tags = [tag for w, tag in nltk.pos_tag(terms)]
if set(['NNS', 'NNPS']) & set(pos_tags):
pred_types = ['List']
pred_types = set(pred_types)
included_types = set(true_types) - set(['NoneType'])
if 'Tuple' in included_types:
included_types -= set(['Tuple'])
included_types.add('List')
# calculate Jaccard Coefficient
jac_list.append(jaccard(pred_types, included_types))
ham_list.append(hamming(pred_types, included_types))
prec_list.append(precision(pred_types, included_types))
rec_list.append(recall(pred_types, included_types))
f1_list.append(f1_score(pred_types, included_types))
acc_list.append(accuracy(pred_types, included_types))
return jac_list, ham_list, prec_list, rec_list, f1_list, acc_list
def train():
print(" Number of training samples %d" % len(train_ind))
print(" Start training...\r\n")
acc = 0
for epoch in range(opt.num_iter):
model.train()
optimizer.zero_grad()
with torch.set_grad_enabled(True):
node_logits, edge_weights = model(features_cuda,
edge_index,
edgenet_input)
loss = loss_fn(node_logits[train_ind], labels[train_ind])
loss.backward()
optimizer.step()
correct_train, acc_train = accuracy(
node_logits[train_ind].detach().cpu().numpy(),
y[train_ind])
model.eval()
with torch.set_grad_enabled(False):
node_logits, _ = model(features_cuda, edge_index,
edgenet_input)
logits_test = node_logits[test_ind].detach().cpu().numpy()
correct_test, acc_test = accuracy(logits_test, y[test_ind])
auc_test = auc(logits_test, y[test_ind])
prf_test = prf(logits_test, y[test_ind])
print("Epoch: {},\tce loss: {:.5f},\ttrain acc: {:.5f}".format(
epoch, loss.item(), acc_train.item()))
if acc_test > acc and epoch > 9:
acc = acc_test
correct = correct_test
aucs[fold] = auc_test
prfs[fold] = prf_test
if opt.ckpt_path != '':
if not os.path.exists(opt.ckpt_path):
#print("Checkpoint Directory does not exist! Making directory {}".format(opt.ckpt_path))
os.makedirs(opt.ckpt_path)
torch.save(model.state_dict(), fold_model_path)
accs[fold] = acc
corrects[fold] = correct
print("\r\n => Fold {} test accuacry {:.5f}".format(fold, acc))
def distill_train(all_logits, dur, epoch, model, optimizer, conf, G,
labels_init, labels, idx_no_train, idx_train, idx_val,
idx_test, cas):
t0 = time.time()
model.train()
optimizer.zero_grad()
if conf['model_name'] in ['GCN', 'APPNP', 'LogReg', 'MLP']:
logits = model(G.ndata['feat'])
elif conf['model_name'] == 'GAT':
logits = model(G.ndata['feat'])[0]
elif conf['model_name'] == 'GraphSAGE':
logits = model(G, G.ndata['feat'])
elif conf['model_name'] == 'PLP':
logits = model(G.ndata['feat'], labels_init)[0]
logp = F.log_softmax(logits, dim=1)
# we only compute loss for labeled nodes
if conf['model_name'] == 'PLP':
loss = my_loss(logp[idx_no_train], cas[-1][idx_no_train])
else:
loss = F.kl_div(logp, cas[-1], reduction='batchmean')
acc_train = accuracy(logp[idx_train], labels[idx_train])
loss.backward()
optimizer.step()
dur.append(time.time() - t0)
model.eval()
if conf['model_name'] in ['GCN', 'APPNP', 'LogReg', 'MLP']:
logits = model(G.ndata['feat'])
elif conf['model_name'] == 'GAT':
logits = model(G.ndata['feat'])[0]
elif conf['model_name'] == 'GraphSAGE':
logits = model(G, G.ndata['feat'])
elif conf['model_name'] == 'PLP':
logits = model(G.ndata['feat'], labels_init)[0]
logp = F.log_softmax(logits, dim=1)
all_logits.append(logp.cpu().detach().numpy())
loss_val = my_loss(logp[idx_val], cas[-1][idx_val])
acc_val = accuracy(logp[idx_val], labels[idx_val])
acc_test = accuracy(logp[idx_test], labels[idx_test])
print(
'Epoch %d | Loss: %.4f | loss_val: %.4f | acc_train: %.4f | acc_val: %.4f | acc_test: %.4f | Time(s) %.4f'
% (epoch, loss.item(), loss_val.item(), acc_train.item(),
acc_val.item(), acc_test.item(), dur[-1]))
return acc_val, loss_val
def valid(net, optimizer, loss, valid_loader, save_imgs=False, fold_num=0):
net.eval()
#keep track of preds
val_preds, val_labels = generate_preds(net, valid_loader, attn=True)
epoch_vloss = loss(val_preds, val_labels)
epoch_vf1 = macro_f1(val_preds.numpy() > 0., val_labels.numpy())
epoch_vacc = accuracy(val_preds.numpy() > 0., val_labels.numpy())
print(
'Avg Eval Loss: {:.4}, Avg Eval Macro F1: {:.4}, Avg Eval Acc. {:.4}'.
format(epoch_vloss, epoch_vf1, epoch_vacc))
return epoch_vloss, epoch_vf1
def validate_model(model,
dataloader,
loss_fn,
is_lstm,
predictions_saver=None,
use_cuda=False,
verbose=False):
# set the model to evaluation mode
model.eval()
top1 = AverageMeter()
count = 0
for batch_idx, (X, y) in enumerate(tqdm(dataloader)):
batch_size = -1
# Utilize GPU if enabled
if use_cuda:
if is_lstm:
X['X'] = X['X'].cuda()
else:
X = X.cuda()
y = y.cuda(async=True)
if is_lstm:
batch_size = X['X'].size(0)
else:
batch_size = X.size(0)
# compute output
predictions = model(X)
count += predictions.shape[0]
loss = loss_fn(predictions, y)
if verbose:
print('Valid/Test Progress [{0}/{1} ({2:.0f}%)]\tLoss:{3}'.format(
count, len(dataloader.dataset),
100. * batch_idx / len(dataloader), loss.item()))
if is_lstm and predictions_saver:
prediction_indices = torch.argmax(predictions, dim=1)
for batch_index in range(batch_size):
predictions_saver.update([
#X['video_dirs'][batch_index].strip('/').split('/')[-1], # id
'replace me',
y.cpu().numpy()[batch_index]
if use_cuda else y.numpy()[batch_index], # label
prediction_indices.cpu().numpy()[batch_index] if use_cuda
else prediction_indices.numpy()[batch_index] # prediction
])
# measure accuracy
acc1 = accuracy(predictions.data, y, (1, ))
top1.update(acc1[0], batch_size)
return top1.avg
def compare_between_models():
tweets = load_anonymized_sentiment_tweets()
print("Tweets:", tweets[:2])
classifier = Classifier(unigrams=True,
bigrams=True,
classifier_type=Type.BOW,
tfidf=False)
act, bow_pred = classifier.ten_fold_cross_validation(tweets)
print("BOW")
metrics.accuracy(act, bow_pred)
classifier = Classifier(unigrams=True,
bigrams=True,
classifier_type=Type.NB,
tfidf=False)
act, nb_pred = classifier.ten_fold_cross_validation(tweets)
print("NB")
metrics.accuracy(act, nb_pred)
classifier = Classifier(unigrams=True,
bigrams=True,
classifier_type=Type.SVM,
tfidf=False)
act, svm_pred = classifier.ten_fold_cross_validation(tweets)
print("SVM")
metrics.accuracy(act, svm_pred)
print("BOW and NB")
metrics.permutation_test(act, bow_pred, nb_pred)
print("BOW and SVM")
metrics.permutation_test(act, bow_pred, svm_pred)
print("NB and SVM")
metrics.permutation_test(act, nb_pred, svm_pred)
def evaluate():
print(" Number of testing samples %d" % len(test_ind))
print(' Start testing...')
model.load_state_dict(torch.load(fold_model_path))
model.eval()
node_logits, _ = model(features_cuda, edge_index, edgenet_input)
logits_test = node_logits[test_ind].detach().cpu().numpy()
corrects[fold], accs[fold] = accuracy(logits_test, y[test_ind])
aucs[fold] = auc(logits_test, y[test_ind])
prfs[fold] = prf(logits_test, y[test_ind])
print(" Fold {} test accuracy {:.5f}, AUC {:.5f}".format(
fold, accs[fold], aucs[fold]))
def run_epoch(model, loader, loss_fn, optimizer, desc_default='', epoch=0, writer=None, verbose=1, scheduler=None):
if verbose:
loader = tqdm(loader, disable=False)
loader.set_description('[{} {}/{}]'.format(desc_default, epoch, C.get()['epoch']))
metrics = Accumulator()
cnt = 0
total_steps = len(loader)
steps = 0
for data, label in loader:
steps += 1
data, label = data.cuda(), label.cuda()
if optimizer:
optimizer.zero_grad()
preds = model(data)
loss = loss_fn(preds, label)
if optimizer:
loss.backward()
if C.get()['optimizer'].get('clip', 5) > 0:
nn.utils.clip_grad_norm_(model.parameters(), C.get()['optimizer'].get('clip', 5))
optimizer.step()
top1, top5 = accuracy(preds, label, (1, 5))
metrics.add_dict({
'loss': loss.item() * len(data),
'top1': top1.item() * len(data),
'top5': top5.item() * len(data),
})
cnt += len(data)
if verbose:
postfix = metrics / cnt
if optimizer:
postfix['lr'] = optimizer.param_groups[0]['lr']
loader.set_postfix(postfix)
if scheduler is not None:
scheduler.step(epoch - 1 + float(steps) / total_steps)
del preds, loss, top1, top5, data, label
metrics /= cnt
if optimizer:
metrics.metrics['lr'] = optimizer.param_groups[0]['lr']
if verbose:
for key, value in metrics.items():
writer.add_scalar(key, value, epoch)
return metrics
def validate(cur_gpu, val_loader, model, criterion, epoch, hparams):
logger = get_logger()
model.eval()
if logger:
loss_meter = AverageMeter('val_loss')
acc1_meter = AverageMeter('val_acc1')
acc5_meter = AverageMeter('val_acc5')
model_module = model.module if hparams.distributed_mode == 'gpus' else model
for i, (image, target) in enumerate(val_loader):
with torch.no_grad():
if cur_gpu >= 0:
image = image.cuda(cur_gpu, non_blocking=True)
target = target.cuda(cur_gpu, non_blocking=True)
if hparams.fp16:
image = image.half()
output = model(image)
if hparams.fp16:
output = output.float()
loss = criterion(output,
target,
label_smoothing=hparams.label_smoothing)
acc1, acc5 = accuracy(output, target, topk=(1, 5))
bs = torch.tensor(image.size(0),
device='cuda:%d' %
cur_gpu if cur_gpu >= 0 else None)
if hparams.distributed_mode == 'gpus':
bs, loss, acc1, acc5 = batch_reduce(bs, loss, acc1, acc5)
if logger:
loss_meter.update(loss.item(), bs.item())
acc1_meter.update(acc1.item(), bs.item())
acc5_meter.update(acc5.item(), bs.item())
loss, acc1, acc5 = None, None, None
if logger:
metrics = [('val_loss', loss_meter.result),
('val_acc1', acc1_meter.result),
('val_acc5', acc5_meter.result)]
logger.log_metrics(metrics, epoch + 1, 0, 'val')
logger.log_summaries(model_module.get_summaries(), epoch + 1, 0, 'val')
loss, acc1, acc5 = loss_meter.result, acc1_meter.result, acc5_meter.result
return loss, acc1, acc5
def validate_eval(val_loader, model, criterion, args, epoch=None, fnames=[]):
batch_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
model.eval()
end = time.time()
scores = np.zeros((len(val_loader.dataset), args.num_class))
labels = np.zeros((len(val_loader.dataset), ))
for i, (frames, objects, target) in enumerate(val_loader):
with torch.no_grad():
target = target.cuda(async=True)
frames = frames.cuda()
objects = objects.cuda()
output = model(frames, objects)
loss = criterion(output, target)
losses.update(loss.item(), target.size(0))
prec1 = accuracy(output.data, target)
top1.update(prec1[0], target.size(0))
batch_time.update(time.time() - end)
end = time.time()
# Record scores.
output_f = F.softmax(output, dim=1) # To [0, 1]
output_np = output_f.data.cpu().numpy()
labels_np = target.data.cpu().numpy()
b_ind = i * args.batch_size
e_ind = b_ind + min(args.batch_size, output_np.shape[0])
scores[b_ind:e_ind, :] = output_np
labels[b_ind:e_ind] = labels_np
print(
'Test [Epoch {0}/{1}]: '
'*Time {2:.2f}mins ({batch_time.avg:.2f}s) '
'*Loss {loss.avg:.4f} '
'*Prec@1 {top1.avg:.3f}'.format(epoch,
args.epoch,
batch_time.sum / 60,
batch_time=batch_time,
top1=top1,
loss=losses))
model.train()
res_scores = multi_scores(scores, labels,
['precision', 'recall', 'average_precision'])
return top1.avg, losses.avg, res_scores['precision'], res_scores[
'recall'], res_scores['average_precision']
def evaluate(self, data, labels):
"""
Args:
data: vector of embeddings
labels: ground truth
Returns: Metrics on the result.
"""
predictions = self.predict(data)
return {
"Accuracy": metrics.accuracy(labels, predictions),
"Recall": metrics.recall(labels, predictions),
"Precision": metrics.precision(labels, predictions),
"F1": metrics.f1(labels, predictions),
"Predictions": predictions
}
def train_one_epoch(self, data_type = "None"):
"""
One epoch of training
:return:
"""
# Initialize tqdm
if data_type == "STRONG":
tqdm_batch = tqdm(self.data_loader.train_SE_loader, total=self.data_loader.train_SE_iterations,
desc="Epoch-{}-".format(self.current_epoch))
print( "Training with SEMEVAL data" )
else:
tqdm_batch = tqdm(self.data_loader.train_loader, total=self.data_loader.train_iterations,
desc="Epoch-{}-".format(self.current_epoch))
print( "Training with All week data and SEMEVAL data" )
self.model.train()
# Initialize your average meters
epoch_loss = AverageMeter()
epoch_acc = AverageMeter()
for itr, (x, y, _) in enumerate(tqdm_batch):
if self.cuda:
x, y = x.cuda(non_blocking=True), y.cuda(non_blocking=True)
# model
pred = self.model(x.permute(1 ,0))
# loss
y = torch.max(y, 1)[1]
y = y.long()
cur_loss = self.loss(pred, y)
if np.isnan(float(cur_loss.item())):
raise ValueError('Loss is nan during training...')
# optimizer
self.optimizer.zero_grad()
cur_loss.backward()
self.optimizer.step()
epoch_loss.update(cur_loss.item())
batch_accuracy = accuracy(y, pred)
epoch_acc.update(batch_accuracy, x.size(0))
self.current_iteration += 1
self.summary_writer.add_scalar("epoch-training/loss", epoch_loss.val, self.current_iteration)
self.summary_writer.add_scalar("epoch_training/accuracy", epoch_acc.val, self.current_iteration)
tqdm_batch.close()
print( "Training Results at epoch-" + str(self.current_epoch) + " | " + "loss: " + str(epoch_loss.val) + " - acc-: " + str(epoch_acc.val ))
def call(self, inputs: list, training: bool = True) -> \
Tuple[tf.Tensor, tf.Tensor]:
"""
Forward propagation
:param inputs: the information passed to next layers
:param training: whether in the training mode
"""
adj_data, u_i, u_j, graph_label, label, idx_mask = inputs
# node level attention
h1 = []
for v in range(self.view_num):
h = self.node_att_layer[v]([self.emb, adj_data[v]])
h = tf.reshape(h, [self.nodes, self.emb.shape[1]])
h1.append(h)
h1 = tf.concat(h1, 0)
h1 = tf.reshape(h1, [self.view_num, self.nodes, self.init_emb_size])
# view level attention
h2 = self.view_att_layer(h1)
a_u = self.olp(h2)
# get masked data
masked_data = tf.gather(a_u, idx_mask)
masked_label = tf.gather(label, idx_mask)
# calculation loss and accuracy
logits = tf.nn.softmax(tf.matmul(masked_data, self.theta))
# Eq. (5)
loss1 = -(1 / len(idx_mask)) * tf.reduce_sum(
masked_label * tf.math.log(tf.nn.softmax(logits)))
u_i_embedding = tf.nn.embedding_lookup(a_u, tf.cast(u_i,
dtype=tf.int32))
u_j_embedding = tf.nn.embedding_lookup(a_u, tf.cast(u_j,
dtype=tf.int32))
inner_product = tf.reduce_sum(u_i_embedding * u_j_embedding, axis=1)
# Eq. (6)
loss2 = -tf.reduce_mean(
tf.math.log_sigmoid(graph_label * inner_product))
# Eq. (7)
loss = self.alpha * loss1 + (1 - self.alpha) * loss2
acc = accuracy(logits, masked_label)
return loss, acc
