def od_kd(epochs, teacher_net, student_net, distiller_net, optimizer, trainloader, testloader, device, **kwargs):
logger = kwargs["logger"]
if "logger_id" not in kwargs:
logger_id = ""
else:
logger_id = kwargs["logger_id"]
scheduler = None
if "scheduler" in kwargs:
scheduler = kwargs["scheduler"]
for epoch in range(1, epochs + 1):
# train for one epoch
loss_ce, total_loss = od_distill_train(device, trainloader, distiller_net, optimizer)
t_acc = eval(teacher_net, device, testloader)
s_acc = eval(student_net, device, testloader)
if logger is not None:
logger.log_scalar("training_loss".format(logger_id), total_loss, epoch)
logger.log_scalar("teacher_val_acc".format(logger_id), t_acc, epoch)
logger.log_scalar("student_val_acc".format(logger_id), s_acc, epoch)
if scheduler:
scheduler.step()
# evaluate on validation set
gc.collect()
def hinton_kd(model, student_model, T, alpha, device, trainloader, testloader,
optimizer, epochs, **kwargs):
scheduler = None
if "scheduler" in kwargs:
scheduler = kwargs["scheduler"]
save_name = ""
if "save_name" in kwargs:
save_name = kwargs["save_name"]
best_acc = 0
for epoch in range(1, epochs + 1):
if scheduler is not None:
scheduler.step()
total_loss = hinton_train(model, student_model, T, alpha, optimizer,
device, trainloader)
t_acc = eval(model, device, testloader)
s_acc = eval(student_model, device, testloader)
if s_acc > best_acc:
best_acc = s_acc
torch.save(
model, "./{}/best_{}_{}.p".format(save_dir, logger_id,
save_name))
return model, optimizer, best_acc
def hinton_without_label(model, student_model, T, device, trainloader, testloader, optimizer, epochs, **kwargs):
logger = kwargs["logger"]
if "logger_id" not in kwargs:
logger_id = ""
else:
logger_id = kwargs["logger_id"]
scheduler = None
if "scheduler" in kwargs:
scheduler = kwargs["scheduler"]
is_debug = False
if "is_debug" in kwargs:
is_debug = kwargs["is_debug"]
best_acc = 0
for epoch in range(1, epochs + 1):
if scheduler is not None:
scheduler.step()
total_loss = hinton_train_without_label(model, student_model, T, optimizer, device, trainloader, is_debug=is_debug)
t_acc = eval(model, device, testloader)
s_acc = eval(student_model, device, testloader)
if logger is not None:
logger.log_scalar("training_loss".format(logger_id), total_loss, epoch)
logger.log_scalar("teacher_val_acc".format(logger_id), t_acc, epoch)
logger.log_scalar("student_val_acc".format(logger_id), s_acc, epoch)
if s_acc > best_acc:
best_acc = s_acc
torch.save(model, "./{}/best_{}.p".format(save_dir, logger_id))
return model, optimizer, best_acc
def do_eval(dep_model, w2i, pos, options, feats=None, feature_type=None):
print "===================================="
print 'Do evaluation'
if not options.sparse_feature:
eval_sentences = utils.read_data(options.dev, True)
else:
eval_sentences = utils.read_sparse_data(options.dev, True,
options.order)
dep_model.eval()
eval_data_list = utils.construct_parsing_data_list(
eval_sentences, w2i, pos, options.length_filter,
options.sparse_feature, options.order, feature_type, feats)
devpath = os.path.join(
options.output,
'test_pred' + str(epoch + 1) + '_' + str(options.sample_idx))
if not options.imbalanced_batch:
eval_batch_data = utils.construct_batch_data(
eval_data_list, options.test_batch_size)
else:
eval_batch_data = utils.construct_imbalanced_batch_data(
eval_data_list, options.test_batch_size, options.order)
for batch_id, one_batch in tqdm(enumerate(eval_batch_data),
mininterval=2,
desc=' -Tot it %d (epoch %d)' %
(len(eval_batch_data), 0),
leave=False,
file=sys.stdout):
if not options.sparse_feature:
eval_batch_words, eval_batch_pos, eval_batch_parent, eval_batch_sen = [s[0] for s in one_batch], \
[s[1] for s in one_batch], \
[s[2] for s in one_batch], \
[s[3][0] for s in one_batch]
eval_batch_words_v = torch.LongTensor(eval_batch_words)
eval_batch_pos_v = torch.LongTensor(eval_batch_pos)
eval_batch_parent_v = torch.LongTensor(eval_batch_parent)
dep_model(eval_batch_words_v, eval_batch_pos_v,
eval_batch_parent_v, eval_batch_sen)
else:
batch_parent = [s[2] for s in one_batch]
batch_feats = [s[4] for s in one_batch]
batch_sen = [s[3][0] for s in one_batch]
batch_feats = torch.LongTensor(batch_feats)
batch_parent_v = torch.LongTensor(batch_parent)
dep_model.forward_sparse(batch_parent_v, batch_feats,
batch_sen)
if options.order == 1:
del dep_model.hm_scores_table
if options.order == 2:
del dep_model.ghm_scores_table
if options.combine_score:
del dep_model.hm_scores_table
if options.order == 3:
del dep_model.ghms_scores_table
if options.gpu > -1 and torch.cuda.is_available():
torch.cuda.empty_cache()
test_res = dep_model.parse_results
utils.eval(test_res, eval_data_list, devpath,
options.log + '_' + str(options.sample_idx), epoch)
print "===================================="
def hinton_without_label(model, student_model, T, device, trainloader,
testloader, optimizer, epochs, **kwargs):
scheduler = None
if "scheduler" in kwargs:
scheduler = kwargs["scheduler"]
is_debug = False
if "is_debug" in kwargs:
is_debug = kwargs["is_debug"]
best_acc = 0
for epoch in range(1, epochs + 1):
if scheduler is not None:
scheduler.step()
total_loss = hinton_train_without_label(model,
student_model,
T,
optimizer,
device,
trainloader,
is_debug=is_debug)
t_acc = eval(model, device, testloader)
s_acc = eval(student_model, device, testloader)
if s_acc > best_acc:
best_acc = s_acc
torch.save(model, "./{}/best_{}.p".format(save_dir, logger_id))
return model, optimizer, best_acc
def do_eval(dep_model, w2i, pos, feats, options):
print "===================================="
print 'Do evaluation on development set'
# unknown_idx = feats["<UNKNOWN-FEATS>"]
# dep_model.feat_param.weight.data[unknown_idx] = 0
eval_sentences = utils.read_data(options.dev, True)
dep_model.eval()
eval_sen_idx = 0
eval_data_list = list()
devpath = os.path.join(options.output, 'test_pred' + str(epoch + 1) + '_' + str(options.sample_idx))
for s in eval_sentences:
s_word, s_pos = s.set_data_list(w2i, pos)
s_data_list = list()
s_data_list.append(s_word)
s_data_list.append(s_pos)
s_data_list.append([eval_sen_idx])
s_feats = utils.construct_feats(feats, s)
s_data_list.append(s_feats)
eval_data_list.append(s_data_list)
eval_sen_idx += 1
eval_batch_data = utils.construct_batch_data(eval_data_list, options.batchsize)
for batch_id, one_batch in enumerate(eval_batch_data):
eval_batch_words, eval_batch_pos, eval_batch_sen, eval_batch_feats = [s[0] for s in one_batch], \
[s[1] for s in one_batch], \
[s[2][0] for s in one_batch], \
[s[3] for s in one_batch]
eval_batch_words_v = utils.list2Variable(eval_batch_words, options.gpu)
eval_batch_pos_v = utils.list2Variable(eval_batch_pos, options.gpu)
eval_batch_feats_v = utils.list2Variable(eval_batch_feats, options.gpu)
dep_model(eval_batch_words_v, eval_batch_pos_v, None, eval_batch_sen, eval_batch_feats_v)
test_res = dep_model.parse_results
utils.eval(test_res, eval_sentences, devpath, options.log + '_' + str(options.sample_idx), epoch)
print "===================================="
def main(argv):
del argv
f = open("eval_output.txt", "a+")
hparams = _default_hparams()
if FLAGS.predict:
predict(FLAGS.input, hparams, path=FLAGS.path)
elif FLAGS.test:
print("Testing")
output = utils.eval(model_dir=FLAGS.model_dir,
model_fn=model_fn,
input_fn=create_test_input_fn,
hparams=hparams,
steps=FLAGS.steps,
batch_size=FLAGS.batch_size,
save_checkpoints_secs=600,
eval_throttle_secs=1800,
eval_steps=5,
sync_replicas=FLAGS.sync_replicas,
task="test",
path=FLAGS.path)
print("Angular median: ", np.median(output['angular_array']))
print("Angular loss: ", output['angular_loss'])
f.close
elif FLAGS.eval:
print("Evaluating")
output = utils.eval(model_dir=FLAGS.model_dir,
model_fn=model_fn,
input_fn=create_test_input_fn,
hparams=hparams,
steps=FLAGS.steps,
batch_size=FLAGS.batch_size,
save_checkpoints_secs=600,
eval_throttle_secs=1800,
eval_steps=5,
sync_replicas=FLAGS.sync_replicas,
task="eval",
path=FLAGS.path)
array_degree = output['angular_array'] * 180 / math.pi
print("Angular median: ", np.median(array_degree))
print("Angular loss: ", output['angular_loss'] * 180 / math.pi)
plt.hist(array_degree, bins=50, range=(0, 180))
plt.show()
f.close
else:
utils.train_and_eval(
model_dir=FLAGS.model_dir,
model_fn=model_fn,
input_fn=create_input_fn,
hparams=hparams,
steps=FLAGS.steps,
batch_size=FLAGS.batch_size,
save_checkpoints_secs=600,
eval_throttle_secs=1800,
eval_steps=5,
sync_replicas=FLAGS.sync_replicas,
)
def main():
##### Setting up datasets #####
# Dictionary:
# Dictionary -- 370k words
# Datasets:
# 1. Wiki - common typos, 4453 words
# 2. Birkbeck - handwritten misspelled words, 34683 words (Randomly sampled at 4453 words)
dictionary = utils.inputDictionary()
(misspells, corrects) = utils.inputDatasets(args)
####### Implementations ########
if args.type == 'soundex':
""" parameters: zero or no zeros."""
zero = True
print("Encoding dictionary and misspells for {0} using {1}...".format(args.dataset, args.type))
dictEncoded = utils.soundex(dictionary, zero)
misEncoded = utils.soundex(misspells, zero)
print("Generating suggestions...")
suggestions = utils.phoneticsSuggestions(misEncoded, dictEncoded, dictionary, args)
precision, recall, numCorrect, totalSuggestions = utils.eval(suggestions, corrects)
resultsOutputStream(args, precision, recall, numCorrect, totalSuggestions, len(misEncoded))
if args.type == 'metaphone':
print("Encoding dictionary and misspells for {0} using {1}...".format(args.dataset, args.type))
dictEncoded = utils.metaphone(dictionary)
misEncoded = utils.metaphone(misspells)
print("Generating suggestions...")
suggestions = utils.phoneticsSuggestions(misEncoded, dictEncoded, dictionary, args)
precision, recall, numCorrect, totalSuggestions = utils.eval(suggestions, corrects)
resultsOutputStream(args, precision, recall, numCorrect, totalSuggestions, len(misEncoded))
if args.type == 'dmetaphone':
print("Encoding dictionary and misspells for {0} using {1}...".format(args.dataset, args.type))
dictEncoded = utils.dMetaphone(dictionary)
misEncoded = utils.dMetaphone(misspells)
assert len(dictEncoded) == len(dictionary) and len(misEncoded) == len(misspells), "Files loaded from pickles are not correct."
print("Generating suggestions...")
suggestions = utils.phoneticsSuggestions(misEncoded, dictEncoded, dictionary, args)
precision, recall, numCorrect, totalSuggestions = utils.eval(suggestions, corrects)
resultsOutputStream(args, precision, recall, numCorrect, totalSuggestions, len(misEncoded))
def evaluate(self, torch_state_dict, update_buffers=False):
criterion = F.cross_entropy
# Recover model from state_dict
self.model.load_state_dict(torch_state_dict)
# Update BatchNorm buffers (if any)
if update_buffers:
utils.bn_update(self.loaders['train'], self.model)
# Evalute on the training and test sets
train_res = utils.eval(self.loaders['train'], self.model, criterion)
test_res = utils.eval(self.loaders['test'], self.model, criterion)
return train_res, test_res
def eval(self,
dset,
vocab=None,
output_false_file=None,
output_label_file=None):
rel_labels = [''] * len(dset.rel2id)
for label, id in dset.rel2id.items():
rel_labels[id] = label
self.model.eval()
pred = []
labels = []
loss = 0.0
acc_cnt = 0.0
trans_mat = np.zeros(
[len(self.rel2id), len(self.rel2id)], dtype=np.int32)
pred_distr = np.ones(len(self.rel2id), dtype=np.float32)
for idx, batch in enumerate(tqdm(dset.batched_data)):
scores_b, pred_b, labels_b, loss_b = self.predict(batch)
pred += pred_b
labels += labels_b
loss += loss_b
for j in range(len(labels_b)):
trans_mat[labels_b[j]][pred_b[j]] += 1
pred_distr[pred_b[j]] += 1
if pred_b[j] == labels_b[j]:
acc_cnt += 1
pred_distr = np.log(pred_distr)
max_log = np.max(pred_distr)
pred_distr = pred_distr - max_log
loss /= len(dset.batched_data)
return loss, utils.eval(
pred, labels), trans_mat, pred_distr, acc_cnt / len(pred)
def btrain(model, device, trainloader, testloader, optimizer, epochs, **kwargs):
logger = kwargs["logger"]
if not "logger_id" in kwargs:
logger_id = ""
else:
logger_id = kwargs["logger_id"]
scheduler = None
if "scheduler"in kwargs:
scheduler = kwargs["scheduler"]
for epoch in range(1, epochs + 1):
if scheduler is not None:
scheduler.step()
start = time.clock()
total_loss = train(model, optimizer, device, trainloader)
end = time.clock()
acc = eval(model, device, testloader)
if logger is not None:
logger.log_scalar("baseline_{}_epoch_time".format(logger_id), time.clock() - end, epoch)
logger.log_scalar("baseline_{}_training_loss".format(logger_id), total_loss, epoch)
logger.log_scalar("baseline_{}_before_target_val_acc".format(logger_id), acc, epoch)
return model, optimizer
def validate(model, val_loader, criterion, params):
model.eval()
full_preds, full_labels = [], []
with torch.no_grad():
val_loss = 0
val_size = 0
for batch, batch_data in enumerate(val_loader, 1):
features, feature_lens, labels, _ = batch_data
batch_size = features.size(0)
# move to gpu if use gpu
if params.gpu is not None:
model.cuda()
features = features.cuda()
feature_lens = feature_lens.cuda()
labels = labels.cuda()
preds = model(features, feature_lens)
# cal loss
loss = 0.0
for i in range(len(params.loss_weights)):
branch_loss = criterion(preds[:, :, i], labels[:, :, i],
feature_lens, params.label_smooth)
loss = loss + params.loss_weights[i] * branch_loss
val_loss += loss.item() * batch_size
val_size += batch_size
full_preds.append(preds.cpu().detach().squeeze(0).numpy())
full_labels.append(labels.cpu().detach().squeeze(0).numpy())
val_loss /= val_size
val_ccc, val_pcc, val_rmse = utils.eval(full_preds, full_labels)
return val_loss, val_ccc, val_pcc, val_rmse
def eval(self,
dset,
vocab=None,
output_false_file=None,
output_label_file=None,
weights=None):
if weights is None:
weights = [1.0] * len(dset.rel2id)
rel_labels = [''] * len(dset.rel2id)
for label, id in dset.rel2id.items():
rel_labels[id] = label
self.model.eval()
pred = []
labels = []
loss = 0.0
for idx, batch in enumerate(dset.batched_data):
scores_b, pred_b, labels_b, loss_b = self.predict(batch)
pred += pred_b
labels += labels_b
loss += loss_b
if output_false_file is not None and vocab is not None:
all_words, pos, ner, subj_pos, obj_pos, labels_, _ = batch
all_words = all_words.tolist()
output_false_file.write('\n')
for i, word_ids in enumerate(all_words):
if labels[i] != pred[i]:
length = 0
for wid in word_ids:
if wid != utils.PAD_ID:
length += 1
words = [vocab[wid] for wid in word_ids[:length]]
sentence = ' '.join(words)
subj_words = []
for sidx in range(length):
if subj_pos[i][sidx] == 0:
subj_words.append(words[sidx])
subj = '_'.join(subj_words)
obj_words = []
for oidx in range(length):
if obj_pos[i][oidx] == 0:
obj_words.append(words[oidx])
obj = '_'.join(obj_words)
output_false_file.write(
'%s\t%s\t%s\t%s\t%s\n' %
(sentence, subj, obj, rel_labels[pred[i]],
rel_labels[labels[i]]))
if output_label_file is not None and vocab is not None:
output_label_file.write(json.dumps(pred) + '\n')
output_label_file.write(json.dumps(labels) + '\n')
loss /= len(dset.batched_data)
return loss, utils.eval(pred, labels, weights)
def od_kd_without_label(epochs, distill, optimizer, trainloader, testloader,
device, **kwargs):
best_acc = 0.0
for epoch in range(1, epochs + 1):
# train for one epoch
loss_ce, total_loss = od_distill_train_without_label(
trainloader, distill, optimizer, device)
t_acc = eval(distill.t_net, device, testloader)
s_acc = eval(distill.s_net, device, testloader)
print(
f'epoch : {epoch}, acc : {s_acc}, t_acc : {t_acc}, best_acc : {best_acc}'
)
if (best_acc < s_acc):
best_acc = s_acc
if (epoch == 100 and epoch == 150):
for param_group in optimizer.param_groups:
param_group['lr'] = param_group['lr'] * .1
def attack_all_model(model_path):
acc = {}
for filename in glob.iglob(model_path + '/*mnist*'):
arch, depth, width = utils.pathextract(filename)
model = utils.load_model(arch, depth, width)
model.load_state_dict(torch.load(filename))
key = '%s_depth_%d_width_%d' % (arch, depth, width)
_, acc[key], _ = utils.eval(model, ct, data_loader)
print('%s \t %.2f ' % (key, acc[key]))
def od_mmd_train(init_lr_da, init_lr_kd, epochs, growth_rate, alpha, gamma, init_beta, distils, source_dataloader, targets_dataloader, targets_testloader,
optimizer_das, optimizer_kds, criterion, device, batch_norm, is_scheduler_da=True, is_scheduler_kd=True, scheduler_da=None, scheduler_kd=None, is_cst=True):
total_loss_arr = []
teacher_da_temp_loss_arr = []
kd_temp_loss_arr = []
teacher_target_acc_arr = []
student_target_acc_arr = []
best_student_acc = 0.
best_teacher_acc = 0.
epochs += 1
for epoch in range(1, epochs):
beta = init_beta * torch.exp(growth_rate * (epoch - 1))
beta = beta.to(device)
if (is_scheduler_da):
new_lr_da = init_lr_da / np.power((1 + 10 * (epoch - 1) / epochs), 0.75) # 10*
for optimizer_da in optimizer_das:
adjust_learning_rate(optimizer_da, new_lr_da)
if (is_scheduler_kd):
new_lr_kd = init_lr_kd / np.power((1 + 10 * (epoch - 1) / epochs), 0.75) # 10*
for optimizer_kd in optimizer_kds:
adjust_learning_rate(optimizer_kd, new_lr_kd)
total_loss_1, total_loss_2, teacher_da_temp_loss_1 = od_mmd_one_epoch(epoch, epochs, distils, source_dataloader,
targets_dataloader, optimizer_das, optimizer_kds,
criterion, device,alpha, beta, gamma, batch_norm, is_cst)
students_targets_acc = np.zeros(len(distils))
for i, d in enumerate(targets_testloader):
students_targets_acc[i] = eval(distils[i].s_net, device, d, False)
total_target_acc = students_targets_acc.mean()
print(f'epoch : {epoch}\tacc : {total_target_acc}')
if (total_target_acc > best_student_acc):
best_student_acc = total_target_acc
torch.save({'student_model': distils[0].s_net.state_dict(), 'acc': best_student_acc, 'epoch': epoch},
"./student_model.pth")
if scheduler_da is not None:
scheduler_da.step()
if scheduler_kd is not None:
scheduler_kd.step()
if(epoch == 150 and epoch == 250):
for optimizer_kd in optimizer_kds:
for param_group in optimizer_kd.param_groups:
param_group['lr'] = param_group['lr'] * .1
return best_student_acc
def do_eval(dep_model, w2i, pos, options):
print "===================================="
print 'Do evaluation on development set'
eval_sentences = utils.read_data(options.dev, True)
dep_model.eval()
eval_sen_idx = 0
eval_data_list = list()
devpath = os.path.join(
options.output,
'test_pred' + str(epoch + 1) + '_' + str(options.sample_idx))
for s in eval_sentences:
s_word, s_pos = s.set_data_list(w2i, pos)
s_data_list = list()
s_data_list.append(s_word)
s_data_list.append(s_pos)
s_data_list.append([eval_sen_idx])
if options.use_trigram:
s_trigram = utils.construct_trigram(s_pos, pos)
s_data_list.append(s_trigram)
eval_data_list.append(s_data_list)
eval_sen_idx += 1
eval_batch_data = utils.construct_batch_data(eval_data_list,
options.batchsize)
for batch_id, one_batch in enumerate(eval_batch_data):
eval_batch_words, eval_batch_pos, eval_batch_sen = [s[0] for s in one_batch], [s[1] for s in one_batch], \
[s[2][0] for s in one_batch]
if options.use_trigram:
batch_trigram = [s[3] for s in one_batch]
batch_trigram_v = utils.list2Variable(batch_trigram,
options.gpu)
else:
batch_trigram_v = None
eval_batch_words_v = utils.list2Variable(eval_batch_words,
options.gpu)
eval_batch_pos_v = utils.list2Variable(eval_batch_pos, options.gpu)
dep_model(eval_batch_words_v, eval_batch_pos_v, None,
eval_batch_sen, batch_trigram_v)
test_res = dep_model.parse_results
utils.eval(test_res, eval_sentences, devpath,
options.log + '_' + str(options.sample_idx), epoch)
print "===================================="
def do_eval(dmv_model, w2i, pos, options):
print "===================================="
print 'Do evaluation on development set'
eval_sentences = utils.read_data(options.dev, True)
dmv_model.eval()
eval_sen_idx = 0
eval_data_list = list()
devpath = os.path.join(
options.output,
'eval_pred' + str(epoch + 1) + '_' + str(options.sample_idx))
for s in eval_sentences:
s_word, s_pos = s.set_data_list(w2i, pos)
s_data_list = list()
s_data_list.append(s_pos)
s_data_list.append(s_word)
s_data_list.append([eval_sen_idx])
eval_data_list.append(s_data_list)
eval_sen_idx += 1
eval_batch_data = utils.construct_batch_data(eval_data_list,
options.batchsize)
parse_results = {}
for batch_id, one_batch in enumerate(eval_batch_data):
eval_batch_pos,eval_batch_words, eval_batch_sen = [s[0] for s in one_batch], [s[1] for s in one_batch], \
[s[2][0] for s in one_batch]
eval_batch_words = np.array(eval_batch_words)
eval_batch_pos = np.array(eval_batch_pos)
batch_score, batch_decision_score = dmv_model.evaluate_batch_score(
eval_batch_words, eval_batch_pos)
if options.function_mask:
batch_score = dmv_model.function_to_mask(
batch_score, eval_batch_pos)
batch_parse = eisner_for_dmv.batch_parse(batch_score,
batch_decision_score,
dmv_model.dvalency,
dmv_model.cvalency)
for i in range(len(eval_batch_pos)):
parse_results[eval_batch_sen[i]] = (batch_parse[0][i],
batch_parse[1][i])
utils.eval(parse_results, eval_sentences, devpath,
options.log + '_dev' + str(options.sample_idx), epoch)
# utils.write_distribution(dmv_model)
print "===================================="
def train(model,
device,
train_loader,
optimizer,
criterion,
epoch,
scheduler,
visualizer=None,
tb_writer=None,
logger=None):
model.train()
train_steps = len(train_loader)
if cfg.TRAIN.STEPS_PER_EPOCH != -1:
train_steps = cfg.TRAIN.STEPS_PER_EPOCH
for batch_idx, (data, target) in enumerate(train_loader):
data, target = data.to(device), target.to(device).long()
optimizer.zero_grad()
output = model(data)
loss = criterion(output, target)
loss.backward()
optimizer.step()
if batch_idx % cfg.TRAIN.LOG_INTERVAL == 0:
tb_idx = batch_idx + epoch * train_steps
if tb_writer is not None:
tb_writer.add_scalar('train_loss', loss.item(), tb_idx)
tb_writer.add_scalar('lr', optimizer.param_groups[0]['lr'],
tb_idx)
if logger is not None:
logger.add_train(loss, tb_idx)
num_samples = batch_idx * cfg.TRAIN.BATCH_SIZE
tot_num_samples = train_steps * cfg.TRAIN.BATCH_SIZE
completed = 100. * batch_idx / train_steps
print(
'Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}\tLR: {:.2e}'.
format(epoch, num_samples, tot_num_samples, completed,
loss.item(), optimizer.param_groups[0]['lr']))
# Evaluate on a fixed test batch for visualization.
if visualizer is not None:
preds = utils.eval(model, visualizer.batch, device)
#visualizer.add_preds(torch.exp(preds).detach(), tb_idx)
# Advance the scheduler at the end of an epoch.
scheduler.step()
return train_steps
def od_kd(epochs, teacher_net, student_net, distiller_net, optimizer,
trainloader, testloader, device, **kwargs):
scheduler = None
if "scheduler" in kwargs:
scheduler = kwargs["scheduler"]
for epoch in range(1, epochs + 1):
# train for one epoch
loss_ce, total_loss = od_distill_train(device, trainloader,
distiller_net, optimizer)
t_acc = eval(teacher_net, device, testloader)
s_acc = eval(student_net, device, testloader)
if scheduler:
scheduler.step()
# evaluate on validation set
gc.collect()
def test_dist():
np.random.seed(0)
p1,p2,p3=(np.random.randn(3,1), np.random.randn(4,1), np.random.randn(5,1))
q1,q2,q3=(np.random.randn(6,1), np.random.randn(7,1), np.random.randn(8,1))
# p1,p2,p3=(np.random.randn(3), np.random.randn(4), np.random.randn(5))
# q1,q2,q3=(np.random.randn(6), np.random.randn(7), np.random.randn(8))
comm = MPI.COMM_WORLD
assert comm.Get_size()==2
if comm.Get_rank()==0:
x1,x2,x3 = p1,p2,p3
elif comm.Get_rank()==1:
x1,x2,x3 = q1,q2,q3
else:
assert False
rms = RunningMeanStd(epsilon=0.0, shape=(1,))
U.initialize()
rms.update(x1)
rms.update(x2)
rms.update(x3)
bigvec = np.concatenate([p1,p2,p3,q1,q2,q3])
def checkallclose(x,y):
print(x,y)
return np.allclose(x,y)
assert checkallclose(
bigvec.mean(axis=0),
U.eval(rms.mean)
)
assert checkallclose(
bigvec.std(axis=0),
U.eval(rms.std)
)
def eval_mean(data_loader):
full_preds_mean, full_labels_mean, full_metas_mean = [], [], []
for batch, batch_data in enumerate(data_loader, 1):
features, _, labels, metas = batch_data
preds = torch.mean(features, 2)
full_preds_mean.append(preds.transpose(0, 1).numpy())
full_labels_mean.append(labels.transpose(0, 1).squeeze(1).numpy())
full_metas_mean.append(metas.numpy())
ccc, pcc, rmse = utils.eval(full_preds_mean, full_labels_mean)
partition = data_loader.dataset.partition
print('For the MEAN of the annotator specific scores:')
print('On {}: CCC: {:7.4f} | PCC: {:7.4f} | RMSE: {:.4f}'.format(
partition, ccc[0], pcc[0], rmse[0]))
def eval_ewe(data_loader):
full_preds, full_labels, full_metas = [], [], []
for batch, batch_data in enumerate(data_loader, 1):
features, _, labels, metas = batch_data
features = features.transpose(1, 2).unsqueeze(-1)
preds = utils.compute_EWE(features.numpy())
preds = list(map(list, zip(*preds))) # transpose list
full_preds.append(preds)
full_labels.append(labels.transpose(0, 1).squeeze(1).numpy())
full_metas.append(metas.numpy())
ccc, pcc, rmse = utils.eval(full_preds, full_labels)
partition = data_loader.dataset.partition
print('For the EWE of the annotator specific scores:')
print('On {}: CCC: {:7.4f} | PCC: {:7.4f} | RMSE: {:.4f}'.format(
partition, ccc[0], pcc[0], rmse[0]))
def validate_quantile_regression(model, val_loader, criterion, params):
model.eval()
full_preds, full_labels = [], []
with torch.no_grad():
val_loss = 0
val_size = 0
for batch, batch_data in enumerate(val_loader, 1):
features, feature_lens, labels, _ = batch_data
batch_size = features.size(0)
# move to gpu if use gpu
if params.gpu is not None:
model.cuda()
features = features.cuda()
feature_lens = feature_lens.cuda()
labels = labels.cuda()
preds = model(features, feature_lens)
# cal loss
loss = 0.0
for i in range(len(params.loss_weights)):
# branch_loss = criterion(preds[:, :, i], labels[:, :, i], feature_lens, params.label_smooth)
branch_loss = criterion(
preds, labels[:, :, i], feature_lens,
params.label_smooth) # NOTE: for tilted loss
loss = loss + params.loss_weights[i] * branch_loss
val_loss += loss.item() * batch_size
val_size += batch_size
# print("Preds:", preds.cpu().detach().squeeze(0).numpy().shape)
# print("Labels:", labels.cpu().detach().squeeze(0).numpy().shape)
# preds = preds.cpu().detach().squeeze(0).numpy()# NOTE: for non-tilted-losses
# preds = preds[:, :, 1:2]# NOTE: use 0.5 quantile for prediction
preds = preds.cpu().detach().squeeze(0).numpy()
preds = preds.mean(axis=1).reshape(
(preds.shape[0],
1)) # NOTE: use mean of all quantiles as prediction
full_preds.append(preds)
full_labels.append(labels.cpu().detach().squeeze(0).numpy())
val_loss /= val_size
val_ccc, val_pcc, val_rmse = utils.eval(full_preds, full_labels)
return val_loss, val_ccc, val_pcc, val_rmse
def train_model(train_dataloader, val_dataloader, model, EPOCHS, BATCH_SIZE, lr, ACCUMULATION_STEPS):
## Optimization
num_train_optimization_steps = int(EPOCHS * len(train_dataloader) / BATCH_SIZE / ACCUMULATION_STEPS)
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(np in n for np in no_decay)], 'weight_decay': 0.01},
{'params': [p for n, p in param_optimizer if any(np in n for np in no_decay)], 'weight_decay': 0.01}
]
optimizer = AdamW(optimizer_grouped_parameters, lr=lr, correct_bias=False)
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=100,
num_training_steps=num_train_optimization_steps)
scheduler0 = get_constant_schedule(optimizer)
frozen = True
# Training
for epoch in (range(EPOCHS+1)):
print("\n--------Start training on Epoch %d/%d" %(epoch, EPOCHS))
avg_loss = 0
avg_accuracy = 0
model.train()
for i, (input_ids, attention_mask, label_batch) in (enumerate(train_dataloader)):
input_ids = input_ids.to(device)
attention_mask = attention_mask.to(device)
label_batch = label_batch.to(device)
optimizer.zero_grad()
y_preds = model(input_ids, attention_mask, None)
loss = torch.nn.functional.binary_cross_entropy(y_preds.to(device),
label_batch.float().to(device))
loss = loss.mean()
loss.backward()
optimizer.step()
lossf = loss.item()
avg_loss += loss.item() / len(train_dataloader)
print("Loss training:", avg_loss)
roc = eval(val_dataloader, model, device)
return model
def test_runningmeanstd():
for (x1, x2, x3) in [
(np.random.randn(3), np.random.randn(4), np.random.randn(5)),
(np.random.randn(3,2), np.random.randn(4,2), np.random.randn(5,2)),
]:
rms = RunningMeanStd(epsilon=0.0, shape=x1.shape[1:])
U.initialize()
x = np.concatenate([x1, x2, x3], axis=0)
ms1 = [x.mean(axis=0), x.std(axis=0)]
rms.update(x1)
rms.update(x2)
rms.update(x3)
ms2 = U.eval([rms.mean, rms.std])
assert np.allclose(ms1, ms2)
def evaluate(model, test_loader, params):
model.eval()
full_preds, full_labels = [], []
with torch.no_grad():
for batch, batch_data in enumerate(test_loader, 1):
features, feature_lens, labels, meta, stdvs = batch_data
if params.gpu is not None:
model.cuda()
features = features.cuda()
feature_lens = feature_lens.cuda()
labels = labels.cuda()
preds = model(features, feature_lens)
full_preds.append(preds.cpu().detach().squeeze(0).numpy())
full_labels.append(labels.cpu().detach().squeeze(0).numpy())
test_ccc, test_pcc, test_rmse = utils.eval(full_preds, full_labels)
return test_ccc, test_pcc, test_rmse
def test():
hparams = _default_hparams()
output = utils.eval(model_dir=args.model_dir,
model_fn=model_fn,
input_fn=create_test_input_fn,
hparams=hparams,
steps=args.steps,
batch_size=args.batch_size,
save_checkpoints_secs=600,
eval_throttle_secs=1800,
eval_steps=5,
sync_replicas=args.sync_replicas,
task="test",
path=args.path)
print("PSNR is: ", output['psnr'])
print("===> Mean PSNR is: ", np.mean(output['psnr']))
print("\n")
def predict(model, dir, output_res=None, batch_size=10, num_classes=6):
order = os.path.basename(dir)[-4]
root = dir.split('/')[0]
writer = utils.wirter_cm(fpath='{}/test-{}.csv'.format(root, order))
class_names = ['VVS', 'VS1', 'VS2', 'SI1', 'SI2']
test_set = Mydataset(img_root=img_path,
txtfile=test_txt,
label_transform=None,
img_transform=val_transform)
loaders = {
'test': DataLoader(
test_set,
batch_size,
shuffle=False,
num_workers=4,
)
}
# model
weight_best = torch.load(dir)['state_dict']
model.load_state_dict(weight_best)
# loss func
criterion = nn.L1Loss()
test_res = utils.eval(loaders['test'], model, criterion, num_classes)
print('test_loss:{:.4f}, test_acc:{:4f}'.format(test_res['loss'],
test_res['accuracy']))
print(test_res['conf_matrix'])
test_res_com = utils.eval_complex(loaders['test'], model, criterion,
num_classes, output_res)
print('test_loss:{:.4f}, test_acc:{:4f}'.format(test_res['loss'],
test_res['accuracy']))
print(test_res_com['conf_matrix'])
infolist = [
os.path.basename(dir), 'test acc', test_res['accuracy'], 'test acc',
test_res_com['accuracy']
]
cms = [test_res['conf_matrix'], test_res_com['conf_matrix']]
writer.writer_in(cms, infolist, class_names)
def scale_grid_search(self,
loader,
criterion,
logscale_range=torch.arange(-10, 0, 0.5).cuda()):
all_losses = torch.zeros_like(logscale_range)
t_s = time.time()
for i, logscale in enumerate(logscale_range):
print('forwards pass with ', logscale)
current_scale = torch.exp(logscale)
self.sample(scale=current_scale)
result = utils.eval(loader, self, criterion)
all_losses[i] = result['loss']
min_index = torch.min(all_losses, dim=0)[1]
scale = torch.exp(logscale_range[min_index]).item()
t_s_final = time.time() - t_s
print('estimating scale took %.2f sec' % (t_s_final))
return scale