def pipeline(nn_model, curr_batch):
# get the inputs (batch)
inputs, labels, lengths, indices = curr_batch
# sort batch (for handling inputs of variable length)
lengths, sort, unsort = sort_batch(lengths)
# sort Variables
inputs = sort(inputs)
labels = sort(labels)
# convert to Variables
inputs = Variable(inputs)
labels = Variable(labels.float())
lengths = Variable(lengths)
# convert to CUDA Variables
if torch.cuda.is_available():
inputs = inputs.cuda(get_gpu_id())
labels = labels.cuda(get_gpu_id())
lengths = lengths.cuda(get_gpu_id())
outputs, attentions = nn_model(inputs, lengths)
# unsort Variables (preserve original order)
outputs = unsort(outputs)
attentions = unsort(attentions)
labels = unsort(labels)
if eval:
return outputs, labels, attentions, None
loss = criterion(outputs, labels)
return outputs, labels, attentions, loss
def get_mask(self, attentions, lengths):
"""
Construct mask for padded itemsteps, based on lengths
"""
max_len = max(lengths.data)
mask = Variable(torch.ones(attentions.size())).detach()
if attentions.data.is_cuda:
mask = mask.cuda(get_gpu_id())
for i, l in enumerate(lengths.data): # skip the first sentence
if l < max_len:
mask[i, l:] = 0
return mask
def classifier(config,
name,
X_train,
X_test,
y_train,
y_test,
ordinal=False,
pretrained=None,
finetune=None,
label_transformer=None,
disable_cache=False):
pretrained_models, pretrained_config = get_pretrained(pretrained)
word2idx = None
if pretrained_config is not None:
_config = pretrained_config
else:
_config = config
if _config["op_mode"] == "word":
word2idx, idx2word, embeddings = load_embeddings(_config)
# construct the pytorch Datasets and Dataloaders
train_set, val_set = load_datasets(X_train,
y_train,
X_test,
y_test,
op_mode=_config["op_mode"],
params=None if disable_cache else name,
word2idx=word2idx,
emojis=_config["emojis"],
label_transformer=label_transformer)
########################################################################
# MODEL
# Define the model that will be trained and its parameters
########################################################################
classes = len(set(y_train))
num_embeddings = None
if _config["op_mode"] == "char":
num_embeddings = len(train_set.char2idx) + 1
embeddings = None
model = GenericModel(embeddings=embeddings,
out_size=1 if classes == 2 else classes,
num_embeddings=num_embeddings,
pretrained=pretrained_models,
finetune=finetune,
**_config)
print(model)
weights = class_weigths(train_set.labels, to_pytorch=True)
if torch.cuda.is_available():
model.cuda(get_new_gpu_id())
weights = weights.cuda(get_gpu_id())
# depending on the number of classes, we should use a different loss
if classes > 2:
criterion = torch.nn.CrossEntropyLoss(weight=weights)
else:
criterion = torch.nn.BCEWithLogitsLoss()
parameters = filter(lambda p: p.requires_grad, model.parameters())
optimizer = torch.optim.Adam(parameters,
weight_decay=config["weight_decay"])
pipeline = pipeline_classification(criterion, binary=classes == 2)
metrics = {
"acc": lambda y, y_hat: accuracy_score(y, y_hat),
"precision":
lambda y, y_hat: precision_score(y, y_hat, average='macro'),
"recall": lambda y, y_hat: recall_score(y, y_hat, average='macro'),
"f1": lambda y, y_hat: f1_score(y, y_hat, average='macro'),
}
monitor = "f1"
trainer = Trainer(model=model,
task="clf",
train_set=train_set,
val_set=val_set,
config=config,
optimizer=optimizer,
pipeline=pipeline,
metrics=metrics,
train_batch_size=config["batch_train"],
eval_batch_size=config["batch_eval"],
use_exp=True,
inspect_weights=False,
checkpoint=Checkpoint(name=name,
model=model,
model_conf=config,
keep_best=True,
scorestamp=True,
metric=monitor,
mode="max",
base=config["base"]),
early_stopping=EarlyStop(metric=monitor,
mode="max",
patience=config["patience"]))
return trainer
def unsort(iterable):
if iterable.is_cuda:
return iterable[reverse_idx.cuda(get_gpu_id())][original_idx.cuda(
get_gpu_id())][reverse_idx.cuda(get_gpu_id())]
else:
return iterable[reverse_idx][original_idx][reverse_idx]
def sort(iterable):
if iterable.is_cuda:
return iterable[sorted_idx.cuda(get_gpu_id())][reverse_idx.cuda(
get_gpu_id())]
else:
return iterable[sorted_idx][reverse_idx]