def plot_active(init=100,
step=100,
idd=100,
suff="active_",
pref="",
coef=None):
train_losses = load_obj(suff + "train_loss_list" + pref)
train_f1s = load_obj(suff + "train_f1_list" + pref)
val_losses = load_obj(suff + "val_loss_list" + pref)
val_f1s = load_obj(suff + "val_f1_list" + pref)
if coef == None:
epoch_dynamic = list(range(init,
len(train_losses) * step + init, step))
else:
epoch_dynamic = [init]
for i in range(len(train_losses) - 1):
epoch_dynamic.append(epoch_dynamic[-1] + epoch_dynamic[-1] * coef)
plt.figure(5 + idd)
plt.ylabel("Loss")
plt.title("Loss during active learning")
plt.xlabel("Dataset len")
plt.plot(epoch_dynamic, train_losses, color="red", label="train set")
plt.plot(epoch_dynamic, val_losses, color="blue", label="valid set")
plt.legend(loc="upper left")
plt.tight_layout()
if coef == None:
plt.savefig("plots/active_loss_i" + str(init) + "_s" + str(step) +
pref + ".png")
else:
plt.savefig("plots/active_loss_i" + str(init) + "_coef.png")
plt.figure(6 + idd)
plt.ylabel("F1")
plt.title("F1 during active learning")
plt.xlabel("Dataset len")
plt.plot(epoch_dynamic, train_f1s, color="red", label="train set")
plt.plot(epoch_dynamic, val_f1s, color="blue", label="valid set")
plt.legend(loc="upper left")
plt.tight_layout()
if coef == None:
plt.savefig("plots/active_f1_i" + str(init) + "_s" + str(step) + pref +
".png")
else:
plt.savefig("plots/active_f1_i" + str(init) + "_coef.png")
def plot_in_comparison(since_epoch):
train_losses = load_obj("train_loss_list")[since_epoch:]
train_f1s = load_obj("train_f1_list")[since_epoch:]
val_losses = load_obj("val_loss_list")[since_epoch:]
val_f1s = load_obj("val_f1_list")[since_epoch:]
total_epochs = len(train_losses) + since_epoch
plt.figure(3)
plt.ylabel("Loss")
plt.title("Loss after " + str(since_epoch) + " epochs")
plt.xlabel("Epochs")
plt.plot(range(since_epoch, total_epochs),
train_losses,
color="red",
label="train set")
plt.plot(range(since_epoch, total_epochs),
val_losses,
color="blue",
label="valid set")
plt.legend(loc="upper left")
plt.tight_layout()
plt.savefig("plots/loss_" + str(since_epoch) + ".png")
plt.figure(4)
plt.ylabel("F1")
plt.title("F1 after " + str(since_epoch) + " epochs")
plt.xlabel("Epochs")
plt.plot(range(since_epoch, total_epochs),
train_f1s,
color="red",
label="train set")
plt.plot(range(since_epoch, total_epochs),
val_f1s,
color="blue",
label="valid set")
plt.legend(loc="lower right")
plt.tight_layout()
plt.savefig("plots/f1_" + str(since_epoch) + ".png")
def evaluate_model(model, dataloader):
model.eval()
predicted_labels = []
labels = []
total_batches = 0
total_loss = 0
with torch.no_grad():
for batch_idx, (chars, toks, lbls, seq_lens) in enumerate(dataloader):
chars = chars.to(device)
toks = toks.to(device)
lbls = lbls.to(device)
emissions = model(chars, toks, seq_lens).to(device)
loss = model.loss(emissions, lbls, seq_lens).item()
total_loss += loss
total_batches += 1
predicted_labels += sum(model.decode(emissions, seq_lens), [])
labels += sum(unpad(lbls.detach().tolist(), seq_lens), [])
final_loss = total_loss / total_batches
idx_to_tag = load_obj("idx_to_tag")
f1, _, _ = score_entities(labels, predicted_labels, idx_to_tag)
return (final_loss, f1)
def plot_last_run():
train_losses = load_obj("train_loss_list")
train_f1s = load_obj("train_f1_list")
val_losses = load_obj("val_loss_list")
val_f1s = load_obj("val_f1_list")
plt.figure(1)
plt.subplot(121)
plt.ylabel("Loss")
plt.title("Loss on training set")
plt.xlabel("Epochs")
plt.plot(train_losses)
plt.subplot(122)
plt.ylabel("F1")
plt.title("F1 on training set")
plt.xlabel("Epochs")
plt.plot(train_f1s)
plt.tight_layout()
plt.savefig("plots/train.png")
plt.figure(2)
plt.subplot(121)
plt.ylabel("Loss")
plt.title("Loss on validation set")
plt.xlabel("Epochs")
plt.plot(val_losses)
plt.subplot(122)
plt.ylabel("F1")
plt.title("F1 on validation set")
plt.xlabel("Epochs")
plt.plot(val_f1s)
plt.tight_layout()
plt.savefig("plots/valid.png")
def plot_mini():
mini_losses = load_obj("mini_loss_list")
mini_f1s = load_obj("mini_f1_list")
plt.figure(1)
plt.subplot(121)
plt.ylabel("Loss")
plt.title("Loss on mini set")
plt.xlabel("Epochs")
plt.plot(mini_losses)
plt.subplot(122)
plt.ylabel("F1")
plt.title("F1 on mini set")
plt.xlabel("Epochs")
plt.plot(mini_f1s)
plt.tight_layout()
plt.savefig("plots/mini.png")
evaluated_entropy += entropy
return evaluated_entropy
if __name__ == '__main__':
limit_memory(7 * 1024 * 1024 * 1024)
seed()
parser = argparse.ArgumentParser()
parser.add_argument("--active", action="store_true")
parser.add_argument("--only-plots", action="store_true")
args = parser.parse_args()
if not args.only_plots:
char_to_idx = load_obj("char_to_idx")
tag_to_idx = load_obj("tag_to_idx")
tok_to_idx = load_obj("tok_to_idx")
if not args.active:
model = load_obj("model")
else:
model = load_obj("active_model")
data_args = dict(
char_to_idx=char_to_idx,
tok_to_idx=tok_to_idx,
tag_to_idx=tag_to_idx,
max_token_len=max_word_len,
preprocessor=preprocess
)
def get_model(self):
return load_obj(self.prefix + "model")
def plot_comparison_active(init=100, step=100, suff="active_"):
pref = "_rand"
train_losses_r = load_obj(suff + "train_loss_list" + pref)
train_f1s_r = load_obj(suff + "train_f1_list" + pref)
val_losses_r = load_obj(suff + "val_loss_list" + pref)
val_f1s_r = load_obj(suff + "val_f1_list" + pref)
train_losses = load_obj(suff + "train_loss_list")
train_f1s = load_obj(suff + "train_f1_list")
val_losses = load_obj(suff + "val_loss_list")
val_f1s = load_obj(suff + "val_f1_list")
total_epochs = len(train_losses) * step + init
plt.figure(7)
plt.ylabel("Loss")
plt.title("Loss during active learning")
plt.xlabel("Dataset len")
plt.plot(range(init, total_epochs, step),
train_losses,
color="red",
label="NSE on train")
plt.plot(range(init, total_epochs, step),
val_losses,
color="blue",
label="NSE on valid")
plt.plot(range(init, total_epochs, step),
train_losses_r,
color="orange",
label="random on train")
plt.plot(range(init, total_epochs, step),
val_losses_r,
color="dodgerblue",
label="random on valid")
plt.legend(loc="upper left")
plt.tight_layout()
plt.savefig("plots/active_comp_loss_i" + str(init) + "_s" + str(step) +
".png")
plt.figure(8)
plt.ylabel("F1")
plt.title("F1 during active learning")
plt.xlabel("Dataset len")
plt.plot(range(init, total_epochs, step),
train_f1s,
color="red",
label="NSE on train")
plt.plot(range(init, total_epochs, step),
val_f1s,
color="blue",
label="NSE on valid")
plt.plot(range(init, total_epochs, step),
train_f1s_r,
color="orange",
label="random on train")
plt.plot(range(init, total_epochs, step),
val_f1s_r,
color="dodgerblue",
label="random on valid")
plt.legend(loc="lower right")
plt.tight_layout()
plt.savefig("plots/active_comp_f1_i" + str(init) + "_s" + str(step) +
".png")