def _init_experiment(self):
"""
init the experiment,
which will visualize and log the performance of the model
Returns:
"""
# 1 - define tags
tags = []
if self.eval_train:
self._init_experiment_tag(self.train_set, tags, "train")
if self.val_set is not None:
self._init_experiment_tag(self.val_set, tags, "val")
# 2 - define experiment
experiment = Experiment(name=self.config["name"],
desc=str(self.model),
hparams=self.config)
# 3 - define metrics
for name, metric in self.metrics.items():
experiment.add_metric(Metric(name=name, tags=tags,
vis_type="line"))
experiment.add_metric(
Metric(name="loss", tags=tags, vis_type="line"))
return experiment
def bcn(config, data_file, embeddings, device, dataset, embeddings_type):
# extensions : add 2 languages, use a combination of CoVe embeddings (like ELMo)
name = "test_model"
torch.manual_seed(123)
inputs = data.Field(lower=True, include_lengths=True, batch_first=True)
labels = data.Field(sequential=False, unk_token=None)
print('Generating train, dev, test splits')
if dataset == 'IWSLT':
# using the IWSLT 2016 TED talk translation task
train, dev, test = datasets.IWSLT.splits(root=data_file,
exts=['.en', '.de'],
fields=[inputs, inputs])
elif dataset == 'SST-2':
train, dev, test = datasets.SST.splits(
text_field=inputs,
label_field=labels,
root=data_file,
fine_grained=False,
train_subtrees=True,
filter_pred=lambda ex: ex.label != 'neutral')
elif dataset == 'SST-5':
train, dev, test = datasets.SST.splits(text_field=inputs,
label_field=labels,
root=data_file,
fine_grained=True,
train_subtrees=True)
elif dataset == 'IMDB':
train, test = datasets.IMDB.splits(text_field=inputs,
label_field=labels,
root=data_file)
train, dev = train.split(
split_ratio=0.9,
stratified=True) # 0.9 in order to be close to the paper
elif dataset == 'TREC-6':
train, test = datasets.TREC.splits(text_field=inputs,
label_field=labels,
root=data_file,
fine_grained=False)
train, dev = train.split(split_ratio=0.9, stratified=True)
elif dataset == 'TREC-50':
train, test = datasets.TREC.splits(text_field=inputs,
label_field=labels,
root=data_file,
fine_grained=True)
train, dev = train.split()
elif dataset == 'SNLI':
train, dev, test = datasets.SNLI.splits(text_field=inputs,
label_field=labels,
root=data_file)
else:
print('Invalid dataset name detected...')
return
print('Building vocabulary')
inputs.build_vocab(train, dev, test)
inputs.vocab.load_vectors(
vectors=GloVe(name='840B', dim=300, cache=embeddings))
labels.build_vocab(train, dev, test)
train_iter, dev_iter, test_iter = data.BucketIterator.splits(
(train, dev, test),
batch_size=config["train_batch_size"],
device=torch.device(device) if device >= 0 else None,
sort_within_batch=True)
model = BCN(config=config,
n_vocab=len(inputs.vocab),
vocabulary=inputs.vocab.vectors,
embeddings=embeddings,
num_labels=len(labels.vocab.freqs),
embeddings_type=embeddings_type)
bcn_params = [
p for n, p in model.named_parameters()
if "mtlstm" not in n and p.requires_grad
]
criterion = nn.CrossEntropyLoss()
optimizer = Adam(bcn_params, lr=0.001)
if device != -1:
model.to(device)
print(model)
total_params = sum(p.numel() for p in model.parameters())
total_trainable_params = sum(p.numel() for p in bcn_params
if p.requires_grad)
print("Total Params:", number_h(total_params))
print("Total Trainable Params:", number_h(total_trainable_params))
#####################################
# Training Pipeline
#####################################
trainer = BCNTrainer(model=model,
train_loader=train_iter,
valid_loader=dev_iter,
criterion=criterion,
device="cpu" if device == -1 else 'cuda',
config=config,
optimizers=[optimizer])
print('Generating CoVe')
####################################################################
# Experiment: logging and visualizing the training process
####################################################################
exp = Experiment(name, config, src_dirs=None, output_dir=EXP_DIR)
exp.add_metric("ep_loss", "line", "epoch loss class", ["TRAIN", "VAL"])
exp.add_metric("ep_f1", "line", "epoch f1", ["TRAIN", "VAL"])
exp.add_metric("ep_acc", "line", "epoch accuracy", ["TRAIN", "VAL"])
exp.add_value("epoch", title="epoch summary")
exp.add_value("progress", title="training progress")
####################################################################
# Training Loop
####################################################################
best_loss = None
early_stopping = EarlyStopping("min", config["patience"])
for epoch in range(1, config["epochs"] + 1):
train_loss = trainer.train_epoch()
print(model.w, model.gama)
val_loss, y, y_pred = trainer.eval_epoch()
# Calculate accuracy and f1-macro on the evaluation set
exp.update_metric("ep_loss", train_loss.item(), "TRAIN")
exp.update_metric("ep_loss", val_loss.item(), "VAL")
exp.update_metric("ep_f1", 0, "TRAIN")
exp.update_metric("ep_f1", f1_macro(y, y_pred), "VAL")
exp.update_metric("ep_acc", 0, "TRAIN")
exp.update_metric("ep_acc", acc(y, y_pred), "VAL")
print()
epoch_log = exp.log_metrics(["ep_loss", "ep_f1", "ep_acc"])
print(epoch_log)
exp.update_value("epoch", epoch_log)
# Save the model if the val loss is the best we've seen so far.
if not best_loss or val_loss < best_loss:
best_loss = val_loss
trainer.best_acc = acc(y, y_pred)
trainer.best_f1 = f1_macro(y, y_pred)
trainer.checkpoint(name=name)
if early_stopping.stop(val_loss):
print("Early Stopping (according to cls loss)....")
break
print("\n" * 2)
return best_loss, trainer.best_acc, trainer.best_f1
def clf_features_baseline_runner(yaml,
word2idx,
idx2word,
weights,
cluster=False):
if cluster is False:
from logger.experiment import Experiment
# torch.manual_seed(0)
# torch.backends.cudnn.deterministic = True
# torch.backends.cudnn.benchmark = False
opts, config = train_options(yaml)
####################################################################
# Data Loading and Preprocessing
####################################################################
X_train, y_train, X_val, y_val, X_test, y_test = load_dataset(
config["data"]["dataset"])
# load word embeddings
# if config["data"]["embeddings"] == "wiki.en.vec":
# word2idx, idx2word, weights = load_word_vectors_from_fasttext(
# os.path.join(EMB_DIR, config["data"]["embeddings"]),
# config["data"]["embeddings_dim"])
# else:
# word2idx, idx2word, weights = load_word_vectors(
# os.path.join(EMB_DIR, config["data"]["embeddings"]),
# config["data"]["embeddings_dim"])
####################################################################
# Linguistic Features Loading and Selection
####################################################################
# Any features/lexicon should be in the form of a dictionary
# For example: lex = {'word':[0., 1., ..., 0.]}
# load affect features
print("Loading linguistic features...")
# todo: streamline feature loading pipeline
features, feat_length = load_features(config["data"]["features"])
# assert ... same len
# build dataset
print("Building training dataset...")
train_set = ClfDataset(X_train,
y_train,
word2idx,
feat_length=feat_length,
features_dict=features)
print("Building validation dataset...")
val_set = ClfDataset(X_test,
y_test,
word2idx,
features_dict=features,
feat_length=feat_length)
test_set = ClfDataset(X_test,
y_test,
word2idx,
features_dict=features,
feat_length=feat_length)
train_set.truncate(500)
val_set.truncate(100)
src_lengths = [len(x) for x in train_set.data]
val_lengths = [len(x) for x in val_set.data]
test_lengths = [len(x) for x in test_set.data]
# select sampler & dataloader
train_sampler = BucketBatchSampler(src_lengths, config["batch_size"], True)
val_sampler = SortedSampler(val_lengths)
val_sampler_train = SortedSampler(src_lengths)
test_sampler = SortedSampler(test_lengths)
train_loader = DataLoader(train_set,
batch_sampler=train_sampler,
num_workers=opts.cores,
collate_fn=ClfCollate_withFeatures())
val_loader = DataLoader(val_set,
sampler=val_sampler,
batch_size=config["batch_size"],
num_workers=opts.cores,
collate_fn=ClfCollate_withFeatures())
val_loader_train_dataset = DataLoader(train_set,
sampler=val_sampler_train,
batch_size=config["batch_size"],
num_workers=opts.cores,
collate_fn=ClfCollate_withFeatures())
test_loader = DataLoader(test_set,
sampler=test_sampler,
batch_size=config["batch_size"],
num_workers=opts.cores,
collate_fn=ClfCollate_withFeatures())
####################################################################
# Model
####################################################################
model = BaselineConcClassifier(ntokens=weights.shape[0],
nclasses=len(set(train_set.labels)),
feat_size=feat_length,
**config["model"])
model.word_embedding.embedding.weight = nn.Parameter(
torch.from_numpy(weights), requires_grad=False)
model.to(opts.device)
print(model)
####################################################################
# Count total parameters of model
####################################################################
total_params = sum(p.numel() for p in model.parameters())
total_trainable_params = sum(p.numel() for p in model.parameters()
if p.requires_grad)
print("Total Params:", number_h(total_params))
print("Total Trainable Params:", number_h(total_trainable_params))
if config["class_weights"]:
class_weights = class_weigths(train_set.labels, to_pytorch=True)
class_weights = class_weights.to(opts.device)
criterion = nn.CrossEntropyLoss(weight=class_weights)
else:
criterion = nn.CrossEntropyLoss()
clf_parameters = filter(lambda p: p.requires_grad, model.parameters())
clf_optimizer = Adam(clf_parameters, weight_decay=1e-5)
####################################################################
# Training Pipeline
####################################################################
_outputs = []
def batch_callback(i_batch, losses, batch_outputs):
_outputs.append(batch_outputs)
if trainer.step % config["log_interval"] == 0:
outputs = list(zip(*_outputs))
_losses = numpy.array(losses[-config["log_interval"]:]).mean(0)
exp.update_metric("clf-loss", _losses)
_y_hat = torch.cat(outputs[0]).max(-1)[1].cpu().data.numpy()
_y = torch.cat(outputs[1]).cpu().data.numpy()
f1 = f1_score(_y, _y_hat, average='macro')
exp.update_metric("f1-train", f1)
losses_log = exp.log_metrics(["clf-loss", 'f1-train'])
exp.update_value("progress",
trainer.progress_log + "\n" + losses_log)
# clean lines and move cursor back up N lines
print("\n\033[K" + losses_log)
print("\033[F" * (len(losses_log.split("\n")) + 2))
_outputs.clear()
# Trainer: responsible for managing the training process
trainer = ClfTrainer_withFeatures(
model=model,
train_loader=train_loader,
valid_loader=val_loader,
valid_loader_train_set=val_loader_train_dataset,
test_loader=test_loader,
criterion=criterion,
optimizers=clf_optimizer,
config=config,
device=opts.device,
batch_end_callbacks=None)
####################################################################
# Experiment: logging and visualizing the training process
####################################################################
if cluster is False:
exp = Experiment(opts.name,
config,
src_dirs=opts.source,
output_dir=EXP_DIR)
exp.add_metric("ep_loss", "line", "epoch loss", ["TRAIN", "VAL"])
exp.add_metric("ep_f1", "line", "epoch f1", ["TRAIN", "VAL"])
exp.add_metric("ep_acc", "line", "epoch accuracy", ["TRAIN", "VAL"])
exp.add_metric("ep_pre", "line", "epoch precision", ["TRAIN", "VAL"])
exp.add_metric("ep_rec", "line", "epoch recall", ["TRAIN", "VAL"])
exp.add_value("epoch", title="epoch summary", vis_type="text")
exp.add_value("progress", title="training progress", vis_type="text")
####################################################################
# Training Loop
####################################################################
best_loss = None
early_stopping = Early_stopping("min", config["patience"])
for epoch in range(config["epochs"]):
train_loss = trainer.train_epoch()
val_loss, y, y_pred = trainer.eval_epoch(val_set=True)
_, y_train, y_pred_train = trainer.eval_epoch(train_set=True)
# Calculate accuracy and f1-macro on the evaluation set
if cluster is False:
exp.update_metric("ep_loss", train_loss.item(), "TRAIN")
exp.update_metric("ep_loss", val_loss.item(), "VAL")
exp.update_metric("ep_f1", f1_macro(y_train, y_pred_train),
"TRAIN")
exp.update_metric("ep_f1", f1_macro(y, y_pred), "VAL")
exp.update_metric("ep_acc", acc(y_train, y_pred_train), "TRAIN")
exp.update_metric("ep_acc", acc(y, y_pred), "VAL")
exp.update_metric("ep_pre", precision_macro(y_train, y_pred_train),
"TRAIN")
exp.update_metric("ep_pre", precision_macro(y, y_pred), "VAL")
exp.update_metric("ep_rec", recall_macro(y_train, y_pred_train),
"TRAIN")
exp.update_metric("ep_rec", recall_macro(y, y_pred), "VAL")
print()
epoch_log = exp.log_metrics(
["ep_loss", "ep_f1", "ep_acc", "ep_pre", "ep_rec"])
print(epoch_log)
exp.update_value("epoch", epoch_log)
exp.save()
else:
print("epoch: {}, train loss: {}, val loss: {}, f1: {}".format(
epoch, train_loss.item(), val_loss.item(), f1_macro(y,
y_pred)))
# Save the model if the validation loss is the best we've seen so far.
if not best_loss or val_loss < best_loss:
best_loss = val_loss
trainer.best_val_loss = best_loss
trainer.acc = acc(y, y_pred)
trainer.f1 = f1_macro(y, y_pred)
trainer.precision = precision_macro(y, y_pred)
trainer.recall = recall_macro(y, y_pred)
trainer.checkpoint(name=opts.name, verbose=False)
if early_stopping.stop(val_loss):
print("Early Stopping...")
break
print("\n")
# return trainer.best_val_loss, trainer.acc, trainer.f1, trainer.precision, trainer.recall
#################
# Test
#################
_, y_test_, y_test_predicted = trainer.eval_epoch(test_set=True)
f1_test = f1_macro(y_test_, y_test_predicted)
acc_test = acc(y_test_, y_test_predicted)
print("#" * 33)
print("F1 for test set: {}".format(f1_test))
print("Accuracy for test set: {}".format(acc_test))
print("#" * 33)
return trainer.best_val_loss, trainer.acc, trainer.f1, trainer.precision, trainer.recall, f1_test, acc_test
def sent_clf(dataset, config, opts, transfer=False):
from logger.experiment import Experiment
opts.name = config["name"]
X_train, y_train, X_val, y_val = dataset
vocab = None
if transfer:
opts.transfer = config["pretrained_lm"]
checkpoint = load_checkpoint(opts.transfer)
config["vocab"].update(checkpoint["config"]["vocab"])
dict_pattern_rename(checkpoint["config"]["model"],
{"rnn_": "bottom_rnn_"})
config["model"].update(checkpoint["config"]["model"])
vocab = checkpoint["vocab"]
####################################################################
# Load Preprocessed Datasets
####################################################################
if config["preprocessor"] == "twitter":
preprocessor = twitter_preprocessor()
else:
preprocessor = None
print("Building training dataset...")
train_set = ClfDataset(X_train,
y_train,
vocab=vocab,
preprocess=preprocessor,
vocab_size=config["vocab"]["size"],
seq_len=config["data"]["seq_len"])
print("Building validation dataset...")
val_set = ClfDataset(X_val,
y_val,
seq_len=train_set.seq_len,
preprocess=preprocessor,
vocab=train_set.vocab)
src_lengths = [len(x) for x in train_set.data]
val_lengths = [len(x) for x in val_set.data]
# select sampler & dataloader
train_sampler = BucketBatchSampler(src_lengths, config["batch_size"], True)
val_sampler = SortedSampler(val_lengths)
val_sampler_train = SortedSampler(src_lengths)
train_loader = DataLoader(train_set,
batch_sampler=train_sampler,
num_workers=opts.cores,
collate_fn=ClfCollate())
val_loader = DataLoader(val_set,
sampler=val_sampler,
batch_size=config["batch_size"],
num_workers=opts.cores,
collate_fn=ClfCollate())
val_loader_train_dataset = DataLoader(train_set,
sampler=val_sampler_train,
batch_size=config["batch_size"],
num_workers=opts.cores,
collate_fn=ClfCollate())
####################################################################
# Model
####################################################################
ntokens = len(train_set.vocab)
model = Classifier(ntokens, len(set(train_set.labels)), **config["model"])
model.to(opts.device)
clf_criterion = nn.CrossEntropyLoss()
lm_criterion = nn.CrossEntropyLoss(ignore_index=0)
embed_parameters = filter(lambda p: p.requires_grad,
model.embed.parameters())
bottom_parameters = filter(
lambda p: p.requires_grad,
chain(model.bottom_rnn.parameters(), model.vocab.parameters()))
if config["model"]["has_att"]:
top_parameters = filter(
lambda p: p.requires_grad,
chain(model.top_rnn.parameters(), model.attention.parameters(),
model.classes.parameters()))
else:
top_parameters = filter(
lambda p: p.requires_grad,
chain(model.top_rnn.parameters(), model.classes.parameters()))
embed_optimizer = optim.ASGD(embed_parameters, lr=0.0001)
rnn_optimizer = optim.ASGD(bottom_parameters)
top_optimizer = Adam(top_parameters, lr=config["top_lr"])
####################################################################
# Training Pipeline
####################################################################
# Trainer: responsible for managing the training process
trainer = SentClfTrainer(model,
train_loader,
val_loader, (lm_criterion, clf_criterion),
[embed_optimizer, rnn_optimizer, top_optimizer],
config,
opts.device,
valid_loader_train_set=val_loader_train_dataset,
unfreeze_embed=config["unfreeze_embed"],
unfreeze_rnn=config["unfreeze_rnn"])
####################################################################
# Experiment: logging and visualizing the training process
####################################################################
exp = Experiment(opts.name,
config,
src_dirs=opts.source,
output_dir=EXP_DIR)
exp.add_metric("ep_loss_lm", "line", "epoch loss lm", ["TRAIN", "VAL"])
exp.add_metric("ep_loss_cls", "line", "epoch loss class", ["TRAIN", "VAL"])
exp.add_metric("ep_f1", "line", "epoch f1", ["TRAIN", "VAL"])
exp.add_metric("ep_acc", "line", "epoch accuracy", ["TRAIN", "VAL"])
exp.add_value("epoch", title="epoch summary")
exp.add_value("progress", title="training progress")
####################################################################
# Resume Training from a previous checkpoint
####################################################################
if transfer:
print("Transferring Encoder weights ...")
dict_pattern_rename(checkpoint["model"], {
"encoder": "bottom_rnn",
"decoder": "vocab"
})
load_state_dict_subset(model, checkpoint["model"])
print(model)
####################################################################
# Training Loop
####################################################################
best_loss = None
early_stopping = EarlyStopping("min", config["patience"])
for epoch in range(0, config["epochs"]):
train_loss = trainer.train_epoch()
val_loss, y, y_pred = trainer.eval_epoch(val_set=True)
_, y_train, y_pred_train = trainer.eval_epoch(train_set=True)
exp.update_metric("ep_loss_lm", train_loss[0], "TRAIN")
exp.update_metric("ep_loss_lm", val_loss[0], "VAL")
exp.update_metric("ep_loss_cls", train_loss[1], "TRAIN")
exp.update_metric("ep_loss_cls", val_loss[1], "VAL")
exp.update_metric("ep_f1", f1_macro(y_train, y_pred_train), "TRAIN")
exp.update_metric("ep_f1", f1_macro(y, y_pred), "VAL")
exp.update_metric("ep_acc", acc(y_train, y_pred_train), "TRAIN")
exp.update_metric("ep_acc", acc(y, y_pred), "VAL")
print()
epoch_log = exp.log_metrics(
["ep_loss_lm", "ep_loss_cls", "ep_f1", "ep_acc"])
print(epoch_log)
exp.update_value("epoch", epoch_log)
# Save the model if the val loss is the best we've seen so far.
if not best_loss or val_loss[1] < best_loss:
best_loss = val_loss[1]
trainer.best_acc = acc(y, y_pred)
trainer.best_f1 = f1_macro(y, y_pred)
trainer.checkpoint(name=opts.name, timestamp=True)
if early_stopping.stop(val_loss[1]):
print("Early Stopping (according to classification loss)....")
break
print("\n" * 2)
return best_loss, trainer.best_acc, trainer.best_f1
def __init__(self,
model,
loaders,
optimizer,
pipeline,
config,
task="clf",
use_exp=False,
inspect_weights=False,
metrics=None,
eval_train=True,
checkpoint=None,
early_stopping=None):
"""
The Trainer is responsible for training a model.
It holds a set of variables that helps us to abstract
the training process.
Args:
use_exp (bool): if True, use the integrated experiment
manager. In order to utilize the visualizations provided
by the experiment manager you should:
- run `python -m visdom.server` in a terminal.
- access visdom by going to http://localhost:8097
https://github.com/facebookresearch/visdom#usage
model (nn.Module): the pytorch model
optimizer ():
pipeline (callable): a callback function, which defines the training
pipeline. it must return 3 things (outputs, labels, loss):
- outputs: the outputs (predictions) of the model
- labels: the gold labels
- loss: the loss
config (): the config instance with the hyperparams of the model
task (string): you can choose between {"clf", "reg"},
for classification and regression respectively.
metrics (dict): a dictionary with the metrics that will be used
for evaluating the performance of the model.
- key: string with the name of the metric.
- value: a callable, with arguments (y, y_hat) tha returns a
score.
eval_train (bool): if True, the at the end of each epoch evaluate
the performance of the model on the training dataset.
early_stopping (EarlyStop):
checkpoint (Checkpoint):
"""
self.use_exp = use_exp
self.inspect_weights = inspect_weights
self.model = model
self.task = task
self.config = config
self.eval_train = eval_train
self.loaders = loaders
self.optimizer = optimizer
self.pipeline = pipeline
self.checkpoint = checkpoint
self.early_stopping = early_stopping
self.metrics = {} if metrics is None else metrics
self.running_loss = 0.0
self.epoch = 0
########################################################
# Initializations
########################################################
_dataset_names = list(self.loaders.keys())
_metric_names = list(self.metrics.keys()) + ["loss"]
# init watched metrics
self.scores = {
metric: {d: []
for d in _dataset_names}
for metric in _metric_names
}
if self.checkpoint is not None:
self.checkpoint.scores = self.scores
if self.early_stopping is not None:
self.early_stopping.scores = self.scores
# init Experiment
if use_exp:
self.experiment = Experiment(name=self.config["name"],
desc=str(self.model),
hparams=self.config)
for metric in _metric_names:
self.experiment.add_metric(
Metric(name=metric, tags=_dataset_names, vis_type="line"))
if self.inspect_weights:
self.inspector = Inspector(model, ["std", "mean"])
class Trainer:
def __init__(self,
model,
loaders,
optimizer,
pipeline,
config,
task="clf",
use_exp=False,
inspect_weights=False,
metrics=None,
eval_train=True,
checkpoint=None,
early_stopping=None):
"""
The Trainer is responsible for training a model.
It holds a set of variables that helps us to abstract
the training process.
Args:
use_exp (bool): if True, use the integrated experiment
manager. In order to utilize the visualizations provided
by the experiment manager you should:
- run `python -m visdom.server` in a terminal.
- access visdom by going to http://localhost:8097
https://github.com/facebookresearch/visdom#usage
model (nn.Module): the pytorch model
optimizer ():
pipeline (callable): a callback function, which defines the training
pipeline. it must return 3 things (outputs, labels, loss):
- outputs: the outputs (predictions) of the model
- labels: the gold labels
- loss: the loss
config (): the config instance with the hyperparams of the model
task (string): you can choose between {"clf", "reg"},
for classification and regression respectively.
metrics (dict): a dictionary with the metrics that will be used
for evaluating the performance of the model.
- key: string with the name of the metric.
- value: a callable, with arguments (y, y_hat) tha returns a
score.
eval_train (bool): if True, the at the end of each epoch evaluate
the performance of the model on the training dataset.
early_stopping (EarlyStop):
checkpoint (Checkpoint):
"""
self.use_exp = use_exp
self.inspect_weights = inspect_weights
self.model = model
self.task = task
self.config = config
self.eval_train = eval_train
self.loaders = loaders
self.optimizer = optimizer
self.pipeline = pipeline
self.checkpoint = checkpoint
self.early_stopping = early_stopping
self.metrics = {} if metrics is None else metrics
self.running_loss = 0.0
self.epoch = 0
########################################################
# Initializations
########################################################
_dataset_names = list(self.loaders.keys())
_metric_names = list(self.metrics.keys()) + ["loss"]
# init watched metrics
self.scores = {
metric: {d: []
for d in _dataset_names}
for metric in _metric_names
}
if self.checkpoint is not None:
self.checkpoint.scores = self.scores
if self.early_stopping is not None:
self.early_stopping.scores = self.scores
# init Experiment
if use_exp:
self.experiment = Experiment(name=self.config["name"],
desc=str(self.model),
hparams=self.config)
for metric in _metric_names:
self.experiment.add_metric(
Metric(name=metric, tags=_dataset_names, vis_type="line"))
if self.inspect_weights:
self.inspector = Inspector(model, ["std", "mean"])
def __on_after_train(self):
pass
def __on_after_eval(self):
# 4 - update the corresponding values in the experiment
if self.use_exp:
for score_name, score in self.scores.items():
for tag, value in score.items():
self.experiment.metrics[score_name].append(tag, value[-1])
self.experiment.update_plots()
if self.inspect_weights:
self.inspector.update_state(self.model)
def train_loader(self, loader):
"""
Run a pass of the model on a given dataloader
Args:
loader ():
Returns:
"""
# switch to train mode -> enable regularization layers, such as Dropout
self.model.train()
running_loss = 0.0
for i_batch, sample_batched in enumerate(loader, 1):
# 1 - zero the gradients
self.optimizer.zero_grad()
# 2 - compute loss using the provided pipeline
outputs, labels, attentions, loss = self.pipeline(
self.model, sample_batched)
# 3 - backward pass: compute gradient wrt model parameters
loss.backward()
# just to be sure... clip gradients with norm > N.
# apply it only if the model has an RNN in it.
if len([
m for m in self.model.modules()
if hasattr(m, 'bidirectional')
]) > 0:
clip_grad_norm_(self.model.parameters(),
self.config["clip_norm"])
# 4 - update weights
self.optimizer.step()
running_loss += loss.item()
# print statistics
epoch_progress(loss=loss.item(),
epoch=self.epoch,
batch=i_batch,
batch_size=loader.batch_size,
dataset_size=len(loader.dataset))
return running_loss
def train(self):
"""
Train the model for one epoch (on one or more dataloaders)
Returns:
"""
self.epoch += 1
self.train_loader(self.loaders["train"])
print()
self.__on_after_train()
def eval_loader(self, loader):
"""
Evaluate a dataloader
and update the corresponding scores and metrics
Args:
loader ():
tag ():
Returns:
"""
# 1 - evaluate the dataloader
avg_loss, (y, y_pred), posteriors, attentions = predict(
self.model, self.pipeline, loader, self.task, "eval")
return avg_loss, (y, y_pred), posteriors, attentions
def eval(self):
"""
Evaluate the model on each dataset and update the corresponding metrics.
The function is normally called at the end of each epoch.
Returns:
"""
for k, v in self.loaders.items():
avg_loss, (y, y_pred), _, _ = self.eval_loader(v)
scores = self.__calc_scores(y, y_pred)
self.__log_scores(scores, avg_loss, k)
scores["loss"] = avg_loss
for name, value in scores.items():
self.scores[name][k].append(value)
self.__on_after_eval()
def __calc_scores(self, y, y_pred):
return {
name: metric(y, y_pred)
for name, metric in self.metrics.items()
}
def __log_scores(self, scores, loss, tag):
"""
Log the scores of a dataset (tag) on the console
Args:
scores (): a dictionary of (metric_name, value)
loss (): the loss of the model on an epoch
tag (): the dataset (name)
Returns:
"""
print("\t{:6s} - ".format(tag), end=" ")
for name, value in scores.items():
print(name, '{:.4f}'.format(value), end=", ")
print(" Loss:{:.4f}".format(loss))
def log_training(self, name, desc):
results = {}
scores = {k: v for k, v in self.scores.items() if k != "loss"}
results["name"] = name
results["desc"] = desc
results["scores"] = scores
path = os.path.join(EXPS_PATH, self.config["name"])
####################################
# JSON
####################################
json_file = path + ".json"
try:
with open(json_file) as f:
data = json.load(f)
except:
data = []
data.append(results)
with open(json_file, 'w') as f:
json.dump(data, f)
####################################
# CSV
####################################
_results = []
for result in data:
_result = {k: v for k, v in result.items() if k != "scores"}
for score_name, score in result["scores"].items():
for tag, values in score.items():
_result["_".join([score_name, tag, "min"])] = min(values)
_result["_".join([score_name, tag, "max"])] = max(values)
_results.append(_result)
with open(path + ".csv", 'w') as f:
pandas.DataFrame(_results).to_csv(f, sep=',', encoding='utf-8')
losses_log)
# clean lines and move cursor back up N lines
print("\n\033[K" + losses_log)
print("\033[F" * (len(losses_log.split("\n")) + 2))
# Trainer: responsible for managing the training process
trainer = LMTrainer(model, train_loader, val_loader, loss_function,
[optimizer], config, opts.device,
batch_end_callbacks=[batch_callback])
####################################################################
# Experiment: logging and visualizing the training process
####################################################################
exp = Experiment(opts.name, config, src_dirs=opts.source, output_dir=EXP_DIR)
exp.add_metric("loss", "line")
exp.add_metric("ppl", "line", "perplexity")
exp.add_metric("ep_loss", "line", "epoch loss", ["TRAIN", "VAL"])
exp.add_metric("ep_ppl", "line", "epoch perplexity", ["TRAIN", "VAL"])
exp.add_value("progress", title="training progress")
exp.add_value("epoch", title="epoch summary")
####################################################################
# Training Loop
####################################################################
best_loss = None
for epoch in range(config["epochs"]):
train_loss = trainer.train_epoch()
val_loss = trainer.eval_epoch()
exp.update_metric("ep_loss", train_loss, "TRAIN")