args = old_args
# clean up a bit
args.pop("checkpoint", None)
print("New args:")
print(args)
# Instantiating the classifier model
print("Building Model")
model = SentimentClassifier(args["freeze_bert"], args["dropout"])
model.cuda(args["gpu"]) # Enable gpu support for the model
print("Creating criterion and optimizer objects")
criterion = nn.BCEWithLogitsLoss()
optimizer = optim.SGD(model.parameters(), lr=args["lr"])
# Creating dataloaders
print("Creating train and val dataloaders")
dataset_train = SSTDataset(filename='data/SST-2/train.tsv',
maxlen=args["maxlen"])
dataset_validation = SSTDataset(filename='data/SST-2/dev.tsv',
maxlen=args["maxlen"])
train_loader = DataLoader(dataset_train,
batch_size=args["batch_size"],
num_workers=5,
shuffle=True,
drop_last=True)
val_loader = DataLoader(dataset_validation,
batch_size=args["batch_size"],
if __name__ == "__main__":
#Get the parameters from arguments if used
parser = ArgumentParser()
parser.add_argument('-freeze_bert', action='store_true')
parser.add_argument('-maxlen', type=int, default=25)
parser.add_argument('-batch_size', type=int, default=32)
parser.add_argument('-lr', type=float, default=2e-5)
parser.add_argument('-print_every', type=int, default=100)
parser.add_argument('-num_eps', type=int, default=5)
args = parser.parse_args()
#Instantiate the classifier model
net = SentimentClassifier(args.freeze_bert)
#CPU or GPU
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
#Put the network to the GPU if available
net = net.to(device)
#Takes as the input the logits of the positive class and computes the binary cross-entropy
criterion = nn.BCEWithLogitsLoss()
#Adam optimizer
optimizer = optim.Adam(net.parameters(), lr=args.lr)
#Create instances of training and validation set
train_set = SSTDataset(filename='data/train.tsv', maxlen=args.maxlen)
val_set = SSTDataset(filename='data/dev.tsv', maxlen=args.maxlen)
#Create intsances of training and validation dataloaders
train_loader = DataLoader(train_set,
batch_size=args.batch_size,
num_workers=5)
val_loader = DataLoader(val_set, batch_size=args.batch_size, num_workers=5)
#Train the network
train(net, criterion, optimizer, train_loader, val_loader, args)
df = pd.read_csv(config['csv_path'])
df_train, df_test = train_test_split(df, test_size=0.1, random_state=RANDOM_SEED)
df_val, df_test = train_test_split(df_test, test_size=0.5, random_state=RANDOM_SEED)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
tokenizer = RobertaTokenizer.from_pretrained(PRE_TRAINED_MODEL_NAME)
train_data_loader = create_data_loader(df_train, tokenizer, MAX_LEN, BATCH_SIZE)
val_data_loader = create_data_loader(df_val, tokenizer, MAX_LEN, BATCH_SIZE)
test_data_loader = create_data_loader(df_test, tokenizer, MAX_LEN, BATCH_SIZE)
model = SentimentClassifier(NUM_CLASSES, PRE_TRAINED_MODEL_NAME)
model = model.to(device)
optimizer = optim.Adam(model.parameters(), lr=lr)
loss_fn = nn.CrossEntropyLoss().to(device)
history = defaultdict(list)
best_accuracy = 0
for epoch in tqdm(range(EPOCHS)):
print(f"Epoch {epoch + 1}/{EPOCHS}")
print("-" * 10)
train_acc, train_loss = train_epoch(
model, train_data_loader, loss_fn, optimizer, device, len(df_train)
)
print(f"Epoch: {epoch}, Train loss: {train_loss}, accuracy: {train_acc}")
BATCH_SIZE = 8
MAX_LEN = 512
train_data_loader = create_data_loader(df_train, tokenizer, MAX_LEN,
BATCH_SIZE)
val_data_loader = create_data_loader(df_val, tokenizer, MAX_LEN, BATCH_SIZE)
test_data_loader = create_data_loader(df_test, tokenizer, MAX_LEN, BATCH_SIZE)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = SentimentClassifier(n_classes=3)
model = model.to(device)
EPOCHS = 10
optimizer = AdamW(model.parameters(), lr=2e-5, correct_bias=False)
total_steps = len(train_data_loader) * EPOCHS
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps=0,
num_training_steps=total_steps)
loss_fn = nn.CrossEntropyLoss().to(device)
history = defaultdict(list)
best_accuracy = 0
for epoch in range(EPOCHS):
print(f'Epoch {epoch + 1}/{EPOCHS}')
print('-' * 10)
def run():
df = pd.read_csv("inputs/reviews.csv")
df["sentiment"] = df.score.apply(rating_to_sentiment)
df_train, df_rem = train_test_split(df,
test_size=0.1,
random_state=config.RANDOM_SEED)
df_val, df_test = train_test_split(df_rem,
test_size=0.5,
random_state=config.RANDOM_SEED)
train_data_loader = create_data_loader(df_train, config.TOKENIZER,
config.MAX_LEN, config.BATCH_SIZE)
val_data_loader = create_data_loader(df_val, config.TOKENIZER,
config.MAX_LEN, config.BATCH_SIZE)
test_data_loader = create_data_loader(df_test, config.TOKENIZER,
config.MAX_LEN, config.BATCH_SIZE)
# data = next(iter(val_data_loader))
# input_ids = data["input_ids"].to(config.DEVICE)
# attention_mask = data["attention_mask"].to(config.DEVICE)
# bert_model = BertModel.from_pretrained(config.BERT_NAME)
model = SentimentClassifier(num_classes=len(class_labels))
if config.LOAD_MODEL == True:
model.load_state_dict(torch.load("best_model_state.bin"))
model = model.to(config.DEVICE)
optimizer = AdamW(model.parameters(), lr=2e-5, correct_bias=False)
total_steps = len(train_data_loader) * config.EPOCHS
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps=0,
num_training_steps=total_steps)
loss_fn = nn.CrossEntropyLoss().to(config.DEVICE)
history = defaultdict(list)
best_accuracy = 0
for epoch in range(config.EPOCHS):
print(f"Epoch {epoch + 1}/{config.EPOCHS}")
print("-" * 10)
train_acc, train_loss = train_fn(
model,
train_data_loader,
loss_fn,
optimizer,
config.DEVICE,
scheduler,
len(df_train),
)
print(f"Train loss {train_loss} accuracy {train_acc}")
val_acc, val_loss = eval_fn(model, val_data_loader, loss_fn,
config.DEVICE, len(df_val))
print(f"Val loss {val_loss} accuracy {val_acc}")
print()
history["train_acc"].append(train_acc)
history["train_loss"].append(train_loss)
history["val_acc"].append(val_acc)
history["val_loss"].append(val_loss)
if val_acc > best_accuracy:
torch.save(model.state_dict(), "best_model_state.bin")
best_accuracy = val_acc