def pipeline(nn_model, curr_batch): # get the inputs (batch) inputs, labels, lengths, indices = curr_batch # sort batch (for handling inputs of variable length) lengths, sort, unsort = sort_batch(lengths) # sort Variables inputs = sort(inputs) labels = sort(labels) # convert to Variables inputs = Variable(inputs) labels = Variable(labels.float()) lengths = Variable(lengths) # convert to CUDA Variables if torch.cuda.is_available(): inputs = inputs.cuda(get_gpu_id()) labels = labels.cuda(get_gpu_id()) lengths = lengths.cuda(get_gpu_id()) outputs, attentions = nn_model(inputs, lengths) # unsort Variables (preserve original order) outputs = unsort(outputs) attentions = unsort(attentions) labels = unsort(labels) if eval: return outputs, labels, attentions, None loss = criterion(outputs, labels) return outputs, labels, attentions, loss
def get_mask(self, attentions, lengths): """ Construct mask for padded itemsteps, based on lengths """ max_len = max(lengths.data) mask = Variable(torch.ones(attentions.size())).detach() if attentions.data.is_cuda: mask = mask.cuda(get_gpu_id()) for i, l in enumerate(lengths.data): # skip the first sentence if l < max_len: mask[i, l:] = 0 return mask
def classifier(config, name, X_train, X_test, y_train, y_test, ordinal=False, pretrained=None, finetune=None, label_transformer=None, disable_cache=False): pretrained_models, pretrained_config = get_pretrained(pretrained) word2idx = None if pretrained_config is not None: _config = pretrained_config else: _config = config if _config["op_mode"] == "word": word2idx, idx2word, embeddings = load_embeddings(_config) # construct the pytorch Datasets and Dataloaders train_set, val_set = load_datasets(X_train, y_train, X_test, y_test, op_mode=_config["op_mode"], params=None if disable_cache else name, word2idx=word2idx, emojis=_config["emojis"], label_transformer=label_transformer) ######################################################################## # MODEL # Define the model that will be trained and its parameters ######################################################################## classes = len(set(y_train)) num_embeddings = None if _config["op_mode"] == "char": num_embeddings = len(train_set.char2idx) + 1 embeddings = None model = GenericModel(embeddings=embeddings, out_size=1 if classes == 2 else classes, num_embeddings=num_embeddings, pretrained=pretrained_models, finetune=finetune, **_config) print(model) weights = class_weigths(train_set.labels, to_pytorch=True) if torch.cuda.is_available(): model.cuda(get_new_gpu_id()) weights = weights.cuda(get_gpu_id()) # depending on the number of classes, we should use a different loss if classes > 2: criterion = torch.nn.CrossEntropyLoss(weight=weights) else: criterion = torch.nn.BCEWithLogitsLoss() parameters = filter(lambda p: p.requires_grad, model.parameters()) optimizer = torch.optim.Adam(parameters, weight_decay=config["weight_decay"]) pipeline = pipeline_classification(criterion, binary=classes == 2) metrics = { "acc": lambda y, y_hat: accuracy_score(y, y_hat), "precision": lambda y, y_hat: precision_score(y, y_hat, average='macro'), "recall": lambda y, y_hat: recall_score(y, y_hat, average='macro'), "f1": lambda y, y_hat: f1_score(y, y_hat, average='macro'), } monitor = "f1" trainer = Trainer(model=model, task="clf", train_set=train_set, val_set=val_set, config=config, optimizer=optimizer, pipeline=pipeline, metrics=metrics, train_batch_size=config["batch_train"], eval_batch_size=config["batch_eval"], use_exp=True, inspect_weights=False, checkpoint=Checkpoint(name=name, model=model, model_conf=config, keep_best=True, scorestamp=True, metric=monitor, mode="max", base=config["base"]), early_stopping=EarlyStop(metric=monitor, mode="max", patience=config["patience"])) return trainer
def unsort(iterable): if iterable.is_cuda: return iterable[reverse_idx.cuda(get_gpu_id())][original_idx.cuda( get_gpu_id())][reverse_idx.cuda(get_gpu_id())] else: return iterable[reverse_idx][original_idx][reverse_idx]
def sort(iterable): if iterable.is_cuda: return iterable[sorted_idx.cuda(get_gpu_id())][reverse_idx.cuda( get_gpu_id())] else: return iterable[sorted_idx][reverse_idx]