コード例 #1
0
def main():
    train_df = pd.read_csv(TRAIN_PATH)
    train_df['male'] = np.load(
        "../input/identity-column-data/male_labeled.npy")
    train_df['female'] = np.load(
        "../input/identity-column-data/female_labeled.npy")
    train_df['homosexual_gay_or_lesbian'] = np.load(
        "../input/identity-column-data/homosexual_gay_or_lesbian_labeled.npy")
    train_df['christian'] = np.load(
        "../input/identity-column-data/christian_labeled.npy")
    train_df['jewish'] = np.load(
        "../input/identity-column-data/jewish_labeled.npy")
    train_df['muslim'] = np.load(
        "../input/identity-column-data/muslim_labeled.npy")
    train_df['black'] = np.load(
        "../input/identity-column-data/black_labeled.npy")
    train_df['white'] = np.load(
        "../input/identity-column-data/white_labeled.npy")
    train_df['psychiatric_or_mental_illness'] = np.load(
        "../input/identity-column-data/psychiatric_or_mental_illness_labeled.npy"
    )
    fold_df = pd.read_csv(FOLD_PATH)

    # y = np.where(train_df['target'] >= 0.5, 1, 0)
    y = train_df['target'].values
    y_aux = train_df[AUX_COLUMNS].values

    identity_columns_new = []
    for column in identity_columns + ['target']:
        train_df[column + "_bin"] = np.where(train_df[column] >= 0.5, True,
                                             False)
        if column != "target":
            identity_columns_new.append(column + "_bin")

    # Overall
    weights = np.ones((len(train_df), )) / 4
    # Subgroup
    weights += (train_df[identity_columns].fillna(0).values >= 0.5).sum(
        axis=1).astype(bool).astype(np.int) / 4
    # Background Positive, Subgroup Negative
    weights += (
        ((train_df["target"].values >= 0.5).astype(bool).astype(np.int) +
         (1 - (train_df[identity_columns].fillna(0).values >= 0.5).sum(
             axis=1).astype(bool).astype(np.int))) > 1).astype(bool).astype(
                 np.int) / 4
    # Background Negative, Subgroup Positive
    weights += (
        ((train_df["target"].values < 0.5).astype(bool).astype(np.int) +
         (train_df[identity_columns].fillna(0).values >= 0.5).sum(
             axis=1).astype(bool).astype(np.int)) > 1).astype(bool).astype(
                 np.int) / 4
    loss_weight = 0.5

    with timer('preprocessing text'):
        # df["comment_text"] = [analyzer_embed(text) for text in df["comment_text"]]
        train_df['comment_text'] = train_df['comment_text'].astype(str)
        train_df = train_df.fillna(0)

    with timer('load embedding'):
        tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH,
                                                  cache_dir=None,
                                                  do_lower_case=True)
        X_text = convert_lines_head_tail(
            train_df["comment_text"].fillna("DUMMY_VALUE"), max_len, head_len,
            tokenizer)
        X_text = np.array(X_text).astype("int32")
        del tokenizer
        gc.collect()

    with timer('train'):
        train_index = fold_df.fold_id != fold_id
        valid_index = fold_df.fold_id == fold_id
        X_train, y_train, y_aux_train, w_train = X_text[train_index], y[
            train_index].astype("float32"), y_aux[train_index].astype(
                "float32"), weights[train_index].astype("float32")
        X_val, y_val, y_aux_val, w_val = X_text[valid_index], y[valid_index].astype("float32"),\
                                         y_aux[valid_index].astype("float32"), weights[valid_index].astype("float32")
        test_df = train_df[valid_index]
        train_size = len(X_train)
        del X_text, y, y_aux, weights, train_index, valid_index, train_df, fold_df
        gc.collect()

        model = BertForSequenceClassification.from_pretrained(
            WORK_DIR, cache_dir=None, num_labels=n_labels)
        model.zero_grad()
        model = model.to(device)

        y_train = np.concatenate(
            (y_train.reshape(-1, 1), w_train.reshape(-1, 1), y_aux_train),
            axis=1).astype("float32")
        y_val = np.concatenate(
            (y_val.reshape(-1, 1), w_val.reshape(-1, 1), y_aux_val),
            axis=1).astype("float32")
        del w_train, w_val, y_aux_train, y_aux_val
        gc.collect()

        train_dataset = torch.utils.data.TensorDataset(
            torch.tensor(X_train, dtype=torch.long),
            torch.tensor(y_train, dtype=torch.float32))
        valid = torch.utils.data.TensorDataset(
            torch.tensor(X_val, dtype=torch.long),
            torch.tensor(y_val, dtype=torch.float32))
        ran_sampler = torch.utils.data.RandomSampler(train_dataset)
        len_sampler = LenMatchBatchSampler(ran_sampler,
                                           batch_size=batch_size,
                                           drop_last=False)
        train_loader = torch.utils.data.DataLoader(train_dataset,
                                                   batch_sampler=len_sampler)
        valid_loader = torch.utils.data.DataLoader(valid,
                                                   batch_size=batch_size * 2,
                                                   shuffle=False)
        del X_train, y_train, X_val, y_val
        gc.collect()
        LOGGER.info(f"done data loader setup")

        param_optimizer = list(model.named_parameters())
        no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
        optimizer_grouped_parameters = [{
            'params': [
                p for n, p in param_optimizer
                if not any(nd in n for nd in no_decay)
            ],
            'weight_decay':
            0.01
        }, {
            'params':
            [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
            'weight_decay':
            0.0
        }]

        num_train_optimization_steps = int(epochs * train_size / batch_size /
                                           accumulation_steps)
        total_step = int(epochs * train_size / batch_size)

        optimizer = BertAdam(optimizer_grouped_parameters,
                             lr=base_lr,
                             warmup=0.005,
                             t_total=num_train_optimization_steps)
        LOGGER.info(f"done optimizer loader setup")

        model, optimizer = amp.initialize(model,
                                          optimizer,
                                          opt_level="O1",
                                          verbosity=0)
        # criterion = torch.nn.BCEWithLogitsLoss().to(device)
        criterion = CustomLoss(loss_weight).to(device)
        LOGGER.info(f"done amp setup")

        for epoch in range(epochs):
            LOGGER.info(f"Starting {epoch} epoch...")
            LOGGER.info(f"length {train_size} train...")
            if epoch == 1:
                for param_group in optimizer.param_groups:
                    param_group['lr'] = base_lr * gammas[1]
            tr_loss, train_losses = train_one_epoch(model,
                                                    train_loader,
                                                    criterion,
                                                    optimizer,
                                                    device,
                                                    accumulation_steps,
                                                    total_step,
                                                    n_labels,
                                                    base_lr,
                                                    gamma=gammas[2 * epoch])
            LOGGER.info(f'Mean train loss: {round(tr_loss,5)}')

            torch.save(model.state_dict(),
                       '{}_epoch{}_fold{}.pth'.format(exp, epoch, fold_id))

            valid_loss, oof_pred = validate(model, valid_loader, criterion,
                                            device, n_labels)
            LOGGER.info(f'Mean valid loss: {round(valid_loss,5)}')

        del model
        gc.collect()
        torch.cuda.empty_cache()

    test_df["pred"] = oof_pred[:, 0]
    test_df = convert_dataframe_to_bool(test_df)
    bias_metrics_df = compute_bias_metrics_for_model(test_df, identity_columns)
    LOGGER.info(bias_metrics_df)

    score = get_final_metric(bias_metrics_df, calculate_overall_auc(test_df))
    LOGGER.info(f'final score is {score}')

    test_df.to_csv("oof.csv", index=False)

    xs = list(range(1, len(train_losses) + 1))
    plt.plot(xs, train_losses, label='Train loss')
    plt.legend()
    plt.xticks(xs)
    plt.xlabel('Iter')
    plt.savefig("loss.png")
コード例 #2
0
device = "cuda:0"
n_labels = len(AUX_COLUMNS) + 1
# n_labels = 1
max_len = 220
head_len = 80
batch_size = 32 * 16
seed_torch(seed)
# ----------------------------------------------------------------------------------------
test['comment_text'] = test['comment_text'].astype(str)
test = test.fillna(0)
# ----------------------------------------------------------------------------------------
tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH,
                                          cache_dir=None,
                                          do_lower_case=True)
X_text = convert_lines_head_tail(
    test['comment_text'].fillna("DUMMY_VALUE").values, max_len, head_len,
    tokenizer)
# ----------------------------------------------------------------------------------------
index = np.argsort(test_lengths)
inverse_index = np.argsort(np.arange(len(test))[index])


# prediction -----------------------------------------------------------------------------
def inference(models, test_loader, device, n_labels):
    test_pred = []
    with torch.no_grad():
        for features in test_loader:
            features = trim_tensors(features[0])
            features = features.to(device)
            for i, model in enumerate(models):
                if i == 0: