def run_cnn(config): headlines_list, labels = create_data_list(config.input_path) bert_embedder = BertEmbedder(headlines_list, labels, config) embeddings, labels = bert_embedder.get_sentence_embeddings() embeddings = embeddings.unsqueeze(0).permute(1, 0, 2) x_train, x_test, y_train, y_test = train_test_split( embeddings, labels, test_size=config.test_size) train_dataset = TensorDataset(x_train, y_train) test_dataset = TensorDataset(x_test, y_test) train_dataloader = DataLoader(train_dataset, sampler=RandomSampler(train_dataset), batch_size=config.batch_size) test_dataloader = DataLoader(test_dataset, sampler=SequentialSampler(test_dataset), batch_size=config.batch_size) # dataset = TensorDataset(embeddings, labels) model = CNNModel(config, checkpoint_file=CNN_CLASSIFIER) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') model.to(device) # train_dataloader, test_dataloader = get_dataloader(dataset, config.test_size, config.batch_size) loss_fn = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=config.lr) fit_result = model.fit(train_dataloader, test_dataloader, loss_fn, optimizer)
def cnn_cross_validation_model(config, n_splits): scores = {} kf = KFold(n_splits=n_splits) params_grid = {'l_r': [3e-5, 5e-5, 1e-4, 1e-3], 'batch_size': [32, 64]} headlines_list, labels = create_data_list(config.input_path) bert_embedder = BertEmbedder(headlines_list, labels, config) print("started embedding") embeddings, labels = bert_embedder.get_sentence_embeddings() print("finished embedding") embeddings = embeddings.unsqueeze(0).permute(1, 0, 2) x_train, x_test, y_train, y_test = train_test_split( embeddings, labels, test_size=config.test_size) max_acc = -numpy.inf for index, params in enumerate(list( ParameterGrid(param_grid=params_grid))): device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print( "-----------------------------------------------------------------" ) print(f"current params are {params}") print(f" index is: {index}") curr_acc = 0 for train_idx, valid_idx in kf.split(x_train): cnn = CNNModel(config, checkpoint_file=CNN_CLASSIFIER) cnn.to(device) cnn.batch_size = params['batch_size'] loss_fn = nn.CrossEntropyLoss() optimizer = optim.Adam(cnn.parameters(), lr=params["l_r"]) train_x, train_y = x_train[train_idx], y_train[train_idx] train_set = TensorDataset(train_x, train_y) valid_x, valid_y = x_train[valid_idx], y_train[valid_idx] valid_set = TensorDataset(valid_x, valid_y) train_dataloader = DataLoader(train_set, sampler=RandomSampler(train_set), batch_size=cnn.batch_size) valid_dataloader = DataLoader(valid_set, sampler=SequentialSampler(valid_set), batch_size=cnn.batch_size) fit_result = cnn.fit(train_dataloader, valid_dataloader, loss_fn, optimizer) curr_acc += max(fit_result.test_acc) mean = curr_acc / n_splits scores[f"{params}"] = mean if mean > max_acc: max_acc = mean best_params = params scores[index] = {"params": params, "acc": mean} print(f"all scores: {scores}") print(f"max accuracy achieved is {max_acc}") print(f"best params are {best_params}") train_dataset = TensorDataset(x_train, y_train) test_dataset = TensorDataset(x_test, y_test) train_dataloader = DataLoader(train_dataset, sampler=RandomSampler(train_dataset), batch_size=best_params['batch_size']) test_dataloader = DataLoader(test_dataset, sampler=SequentialSampler(test_dataset), batch_size=best_params['batch_size']) best_model = CNNModel(config, checkpoint_file=CNN_CLASSIFIER) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') best_model.to(device) best_model.batch_size = best_params['batch_size'] loss_fn = nn.CrossEntropyLoss() optimizer = optim.Adam(best_model.parameters(), lr=best_params["l_r"]) fit_result = best_model.fit(train_dataloader, test_dataloader, loss_fn, optimizer)
args.emb_size, args.max_seq_length, weights=vocab_weights, fix_emb_weight=args.fix_emb) else: model = BILSTMModel(vocab_size, args.emb_size, args.max_seq_length, weights=vocab_weights, fix_emb_weight=args.fix_emb) model = model.to(device) # Train print("Training %s ..." % args.model) params = filter(lambda p: p.requires_grad, model.parameters()) optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) loss_func = nn.CrossEntropyLoss() #label_weight = WeightClassCSV(args.dataset + "/train.csv").get_weights(['0', '1']) #loss_func = nn.CrossEntropyLoss(weight = torch.tensor(label_weight).to(device)) model.train() for epoch in trange(args.num_epoch, desc="Epoch"): total_loss = 0 for idx, batch in enumerate(tqdm(train_iter, desc="Iteration")): inputs, labels = batch.sent, batch.label inputs = inputs.to(device) labels = labels.to(device) logits = model(inputs)