def main(config_path, output_file):
warnings.filterwarnings("ignore", message="numpy.dtype size changed")
warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
config = Box.from_yaml(config_path.open())
logger = create_logger(name="MAIN")
logger.info(f'[-] Config loaded from {config_path}')
vocab = gen_vocab(Path(config.data_dir),
Path(config.data_dir) / "save" / "vocab.pkl",
config.special_tokens, config.vocab_size)
model = Model(config, vocab)
if hasattr(config, "test_ckpt"):
logger.info(f'[-] Test checkpoint: {config.test_ckpt}')
model.load_state(config.test_ckpt)
else:
logger.info(f'[-] Test experiment: {config.exp}')
model.load_best_state(Path(config.data_dir) / "exp" / config.exp)
random.seed(config.random_seed)
np.random.seed(config.random_seed)
torch.manual_seed(config.random_seed)
torch.cuda.manual_seed(config.random_seed)
logger.info('[-] Random seed set to {}'.format(config.random_seed))
logger.info(f'[*] Initialize data loader...')
data_loader = create_data_loader(Path(config.data_dir) / "save" /
"test.pkl",
vocab,
config.p_sent,
config.p_seq_len,
config.q_seq_len,
config.c_seq_len,
config.batch_size,
debug=False)
logger.info('[*] Start testing...')
writer = csv.DictWriter(open(output_file, 'w'), fieldnames=['id', 'ans'])
writer.writeheader()
for batch in data_loader:
batch['passage'] = batch['passage'].to(model._device)
batch['question'] = batch['question'].to(model._device)
batch['choices'] = batch['choices'].to(model._device)
logits = model(batch['passage'], batch['question'], batch['choices'])
_, predictions = torch.max(logits, dim=-1)
predictions = predictions.tolist()
for idx, (ID, pred) in enumerate(zip(batch['id'], predictions)):
question = batch['question'][idx].tolist()
if vocab.word.pad in question:
question = question[:question.index(vocab.word.pad)]
answer = batch['choices'][pred - 1][idx].tolist()
if vocab.word.pad in answer:
answer = answer[:answer.index(vocab.word.pad)]
writer.writerow({'id': ID, 'ans': int(pred) + 1})
print(f"Question: {''.join(vocab.word.t2s(question))}")
print(f"Answer: {''.join(vocab.word.t2s(answer))}")
def __initialize(self):
self.train_bar = tqdm(
range(self.__cur_epoch, self.epochs),
total=self.epochs,
desc='[Total Progress]',
initial=self.__cur_epoch,
position=0)
bar_header = ["EPOCH", "MODE"] + self.metrics
self.train_bar.write(self.__display(bar_header))
log_path = self.exp_dir / "log.csv"
fieldnames = [
f'{mode}_{metric}' for mode, metric
in product(["TRAIN", "VALID"], self.metrics)]
if self.__cur_epoch == 0:
self.__log_writer = csv.DictWriter(
log_path.open(mode='w', buffering=1),
fieldnames=fieldnames)
self.__log_writer.writeheader()
else:
self.__log_writer = csv.DictWriter(
log_path.open(mode='a', buffering=1),
fieldnames=fieldnames)
self.train_data_loader = create_data_loader(
f"{self.data_dir}/train.pkl",
self.vocab,
self.p_sent,
self.p_seq_len,
self.q_seq_len,
self.c_seq_len,
self.batch_size,
getattr(self, "debug", "False"))
self.valid_data_loader = create_data_loader(
f"{self.data_dir}/valid.pkl",
self.vocab,
self.p_sent,
self.p_seq_len,
self.q_seq_len,
self.c_seq_len,
self.batch_size,
getattr(self, "debug", "False"))
def train(
bert_model_name: str,
train_split_path: str,
val_split_path: str,
max_len: int,
train_batch_size: int,
val_batch_size: int,
dropout: float,
linear_units: int,
epochs: int,
classification_model_path: str,
):
df_train = pd.read_csv(train_split_path).fillna("none")
df_val = pd.read_csv(val_split_path).fillna("none")
train_params = {
"batch_size": train_batch_size,
"num_workers": 0,
"bert_model_name": bert_model_name,
"max_len": max_len,
}
test_params = {
"batch_size": val_batch_size,
"num_workers": 0,
"bert_model_name": bert_model_name,
"max_len": max_len,
}
train_dataset = create_data_loader(df_train, **train_params)
test_dataset = create_data_loader(df_val, **test_params)
device = torch.device("cuda:0")
model = BERTBaseUncased(bert_model_name, dropout, linear_units)
model = model.to(device)
loss_function = nn.CrossEntropyLoss()
LEARNING_RATE = 1e-05
optimizer = torch.optim.Adam(params=model.parameters(), lr=LEARNING_RATE)
for epoch in range(epochs):
train_step(epoch, train_dataset, model, loss_function, device, optimizer)
def run(options):
work_dir = os.path.join(options.work_dir, "localizer")
print(" + Random seed: %d" % options.random_seed)
random.seed(options.random_seed)
torch.manual_seed(options.random_seed)
np.random.seed(options.random_seed)
# model = CervixLocalisationModel(num_classes = NUM_CLASSES, batch_norm=True)
model = torchvision.models.resnet18(num_classes = NUM_CLASSES)
criterion = torch.nn.MSELoss()
# criterion= torch.nn.CrossEntropyLoss()
is_cuda = not options.no_cuda and torch.cuda.is_available()
print(" + CUDA enabled" if is_cuda else " - CUDA disabled")
if is_cuda:
torch.cuda.manual_seed(options.random_seed)
model = torch.nn.DataParallel( model ).cuda()
criterion = criterion.cuda()
# optimizer = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=1e-4)
optimizer = torch.optim.Adadelta(model.parameters(), weight_decay=options.l2reg)
if os.path.isfile(options.model_path):
print( " + Loading model: %s" % options.model_path)
model_state_dict = torch.load(options.model_path)
model.load_state_dict(model_state_dict)
print(" + Mini-batch size: %d" % options.batch_size)
if options.train_epochs and options.train_input_path:
print(" + Training epochs : %d" % options.train_epochs)
print(" + input file : %s" % options.train_input_path)
validation_split = options.validation_split if options.validation_split > 0 and options.validation_split < 1 else None
if validation_split:
print(" + validation split: %.2f" % validation_split)
train_loader, validate_loader = dataset.create_data_loader(
annotations_path = options.train_input_path,
transform = transforms.Compose (epochs_transform),
batch_size = options.batch_size,
num_workers = options.workers,
validation_split = validation_split,
is_train = True
)
print(" + training samples: %.d" % len(train_loader.dataset) )
for epoch in range(0, options.train_epochs):
# train on single epoch
train_loss = train_single_epoch(model, criterion, optimizer, train_loader, epoch, is_cuda)
loss_str = "%d: train_loss: %.6f" % (epoch, train_loss)
if validation_split:
val_output, val_targets = evaluate(model, validate_loader, is_cuda)
val_loss = validation_loss(criterion, val_output, val_targets)
loss_str += ", val_loss: %.6f" % val_loss
print(loss_str)
print(" + Training completed")
if not options.dry_run:
print( " + Saving model: %s" % options.model_path)
torch.save( model.state_dict(), options.model_path)
if options.evaluate_input_path:
print(" + Evaluating input file: %s" % options.evaluate_input_path)
eval_loader,_ = dataset.create_data_loader(
annotations_path = options.evaluate_input_path,
transform = transforms.Compose (eval_transform),
batch_size = options.batch_size,
num_workers = options.workers,
validation_split = None,
is_train = False
)
output, targets = evaluate(model, eval_loader, is_cuda)
eval_loss = validation_loss(criterion, output, targets)
print(" + eval_loss: %.6f" % eval_loss)
# Save results
if options.evaluate_output_path:
print(" + Exporting results to: %s" % options.evaluate_output_path)
res = extract_results( eval_loader, output )
save_annotations( res , options.evaluate_output_path)
print(" + DONE")
from model import create_model, load_model, save_model, move_model_to_GPU
from dataset import create_data_loader
from utils import save_training_losses, get_training_losses, calculate_avg_epoch_losses
##############
model_num = 6
##############
# Create the model and use GPU if available
model = create_model(isTrain=True)
#model = load_model("model_"+str(model_num)+"_epoch_2")
model, device = move_model_to_GPU(model)
# Create the data generators
training_loader = create_data_loader("train")
validation_loader = create_data_loader("val")
epoch_count = 500
save_model_every = 5
print_losses_every = 100
losses_G = []
losses_D = []
print("Training the model...")
for epoch in range(epoch_count):
print("Epoch:", epoch + 1)
# Lists to calculate average epoch losses
import torch
import pandas as pd
from dataset import create_data_loader
from model import multimodal
from utils import get_predictions
from zipfile import ZipFile
LOAD_MODEL = True
device = 'cuda'
model = multimodal()
model = model.to(device)
if LOAD_MODEL:
model.load_state_dict(torch.load('vit-bert-1.0val.bin'))
df_test = pd.read_csv('test_captions.csv')
df_test.drop('Unnamed: 0', axis=1, inplace=True)
extract_path = 'test_img.zip'
with ZipFile(extract_path, 'r') as zipObj:
zipObj.extractall()
test_data_loader = create_data_loader(df_test, tokenizer, MAX_LEN, BATCH_SIZE,
my_trans, 'test_img', False)
submission_preds = get_predictions(model, test_data_loader, device)
def run(options):
work_dir = os.path.join(options.work_dir, "classifier")
print(" + Random seed: %d" % options.random_seed)
random.seed(options.random_seed)
torch.manual_seed(options.random_seed)
np.random.seed(options.random_seed)
print(" + Classes: %d" % NUM_CLASSES)
model = CervixClassificationModel(num_classes=NUM_CLASSES, batch_norm=True)
# model = torchvision.models.inception_v3(num_classes = NUM_CLASSES, aux_logits=False)
# model = torchvision.models.vgg16_bn(num_classes = NUM_CLASSES)
criterion = torch.nn.CrossEntropyLoss()
is_cuda = not options.no_cuda and torch.cuda.is_available()
print(" + CUDA enabled" if is_cuda else " - CUDA disabled")
if is_cuda:
torch.cuda.manual_seed(options.random_seed)
model = torch.nn.DataParallel(model).cuda()
criterion = criterion.cuda()
# optimizer = torch.optim.Adagrad(model.parameters(),lr=options.lr, weight_decay=5e-2)
optimizer = torch.optim.SGD(model.parameters(),
lr=options.lr,
momentum=0.9,
weight_decay=0.03)
if os.path.isfile(options.model_path):
print(" + Loading model: %s" % options.model_path)
model_state_dict = torch.load(options.model_path)
model.load_state_dict(model_state_dict)
print(" + Mini-batch size: %d" % options.batch_size)
if options.train_epochs and options.train_input_path:
print(" + Training epochs : %d" % options.train_epochs)
print(" + input file : %s" % options.train_input_path)
validation_split = options.validation_split if options.validation_split > 0 and options.validation_split < 1 else None
train_loader, validate_loader = dataset.create_data_loader(
annotations_path=options.train_input_path,
batch_size=options.batch_size,
num_workers=options.workers,
validation_split=validation_split,
is_train=True,
type_map=TYPE_MAP,
rseed=options.random_seed)
if validation_split:
print(" + validation split: %.2f" % validation_split)
print(" + training samples: %.d" % len(train_loader.sampler))
if validation_split:
print(" + validate samples: %.d" %
len(validate_loader.sampler))
with open(os.path.join(work_dir, "run.log"), 'w') as fp:
for epoch in range(0, options.train_epochs):
adjust_learning_rate(options.lr, optimizer, epoch)
# train on single epoch
train_loss = train_single_epoch(model, criterion, optimizer,
train_loader, epoch, is_cuda)
loss_str = "%d: train_loss: %.6f" % (epoch, train_loss)
if validation_split and epoch % 5 == 0:
val_output, val_targets = evaluate_avg(
model, validate_loader, is_cuda, 5)
val_loss = validation_loss(criterion, val_output,
val_targets)
acc_val = accuracy(val_output, val_targets)
loss_str += ", val_loss: %.6f" % val_loss
loss_str += ", accuracy: %.1f" % (acc_val * 100)
print(loss_str)
#print_worst(epoch, 500, val_images, val_output, val_targets, fp)
#fp.flush()
print(" + Training completed")
if not options.dry_run:
print(" + Saving model: %s" % options.model_path)
torch.save(model.state_dict(), options.model_path)
if options.evaluate_input_path:
print(" + Evaluating input file: %s" % options.evaluate_input_path)
eval_loader, _ = dataset.create_data_loader(
annotations_path=options.evaluate_input_path,
num_workers=options.workers,
batch_size=options.batch_size,
validation_split=None,
is_train=False,
type_map=TYPE_MAP)
output, targets = evaluate_avg(model, eval_loader, is_cuda, 5)
if targets.size():
eval_loss = validation_loss(criterion, output, targets)
print(" + eval_loss: %.6f" % eval_loss)
# Save results
if options.evaluate_output_path:
print(" + Exporting results to: %s" % options.evaluate_output_path)
softmax = torch.nn.Softmax()
output = softmax(torch.autograd.Variable(output)).data
save_evaluation_results(options.evaluate_output_path, eval_loader,
output)
print(" + DONE")
from loss import WeightAdjustingLoss, FocalLoss
from dataset import OffensiveDataset, create_data_loader
from model import OffensiveModel
from collections import defaultdict
device = 'cuda' if torch.cuda.is_available() else 'cpu'
pretrained_name = 'bert-base-multilingual-cased'
plt.style.use("ggplot")
BATCH_SIZE = 32
MAX_LEN = 128
EPOCHS = 4
history = defaultdict(list)
best_accuracy = 0
LOAD_MODEL = False
train_data_loader = create_data_loader(train_df, tokenizer, MAX_LEN, BATCH_SIZE, shuffle=True)
test_data_loader = create_data_loader(test_df, tokenizers, MAX_LEN, BATCH_SIZE, shuffle=False)
val_data_loader = create_data_loader(val_df, tokenizers, MAX_LEN, BATCH_SIZE, shuffle=False)
model = OffensiveModel()
model = model.to(device)
optimizer = AdamW(model.parameters(), lr=2e-5, correct_bias=False)
total_steps = len(train_data_loader) * EPOCHS
scheduler = get_linear_schedule_with_warmup(
optimizer,
num_warmup_steps=0,
num_training_steps=total_steps
)
loss = WeightAdjustingLoss().to(device)
import torch
from model import load_model, move_model_to_GPU
from dataset import create_data_loader
##############
model_num = 0
##############
test_loader = create_data_loader("test")
model = load_model("model_" + str(model_num) + "_epoch_219", isTrain=True) # Optimize et
#model = model.netG
#model.eval()
#print(model.loss_history_G)
model, device = move_model_to_GPU(model)
print("Testing the model...")
for iter, batch in enumerate(test_loader):
batch = batch.to(device)
with torch.no_grad():
model.set_input(batch)
model.forward()
model.save_images(iter, len(batch))
print("Testing done.")
from transformers import AdamW
from model import SentimentClassifier
from dataset import create_data_loader
from utils import train_epoch, eval_model, get_predictions, PRETRAINED_MODEL_NAME
tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)
df_train = pd.read_csv('train.csv')
df_test = pd.read_csv('test.csv')
df_train, df_val = train_test_split(df_train, test_size=0.25)
BATCH_SIZE = 8
MAX_LEN = 512
train_data_loader = create_data_loader(df_train, tokenizer, MAX_LEN,
BATCH_SIZE)
val_data_loader = create_data_loader(df_val, tokenizer, MAX_LEN, BATCH_SIZE)
test_data_loader = create_data_loader(df_test, tokenizer, MAX_LEN, BATCH_SIZE)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = SentimentClassifier(n_classes=3)
model = model.to(device)
EPOCHS = 10
optimizer = AdamW(model.parameters(), lr=2e-5, correct_bias=False)
total_steps = len(train_data_loader) * EPOCHS
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps=0,
def run():
print ("Importing the datasets...\n")
df = pd.read_csv(config.Training_file)
# Multi-Genre NLI Corpus
df_mnli = data_augmentation.load_mnli()
# Cross-Lingual NLI Corpus
df_xnli = data_augmentation.load_xnli()
df.drop(columns = ['id'], inplace = True)
print ("Imported...\n")
print ("Processing and doing Data Augmentation ...\n")
print ("back translation started...\n")
df = data_augmentation.proc(df)
print ("Back Translation Done...\n")
df = pd.concat([df, df_mnli, df_xnli], ignore_index = True)
# Shuffling the dataframe
df = resample(df, random_state = config.RANDOM_SEED)
df['premise'] = df['premise'].astype(str)
df['hypothesis'] = df['hypothesis'].astype(str)
df.drop_duplicates(subset = ['premise', 'hypothesis'], inplace = True)
combined_thesis = df[['premise', 'hypothesis']].values.tolist()
df['combined_thesis'] = combined_thesis
df_train, df_test = train_test_split(
df,
test_size = 0.3,
random_state = 0
)
df_test, df_val = train_test_split(
df_test,
test_size = 0.4,
random_state = 0
)
train_data_loader = dataset.create_data_loader(df_train, config.tokenizer, config.max_len, config.batch_size)
test_data_loader = dataset.create_data_loader(df_test, config.tokenizer, config.max_len, config.batch_size)
val_data_loader = dataset.create_data_loader(df_val, config.tokenizer, config.max_len, config.batch_size)
print ("Processing and Data Augmentation Done...\n")
# we save the whole model of roberta_model in the model var which is further used in the code
device = torch.device(config.DEVICE)
model = roberta_model(3)
model = model.to(device)
#optimizer, scheduler and loss_fn used in the train_roberta_epoch and eval_roberta
optimizer = AdamW(model.parameters(), lr=2e-5, correct_bias=False)
total_steps = len(train_data_loader) * config.EPOCHS
scheduler = get_linear_schedule_with_warmup(
optimizer,
num_warmup_steps=0,
num_training_steps=total_steps
)
loss_fn = nn.CrossEntropyLoss().to(device)
history = defaultdict(list)
best_accuracy = 0
print ("Training...\n")
for epochs in range(config.EPOCHS):
print ('Epoch {} \n'.format(epochs + 1))
print ("-"*100)
train_acc, train_loss = engine.train_roberta_epoch(
model,
train_data_loader,
loss_fn,
optimizer,
device,
scheduler,
len(df_train)
)
print ('train_acc {} train_loss {}'.format(train_acc, train_loss))
val_acc, val_loss = engine.eval_roberta(
model,
val_data_loader,
loss_fn,
device,
len(df_val)
)
print ('val_acc {} val_loss {}'.format(val_acc, val_loss))
print ()
history['train_acc'].append(train_acc)
history['train_loss'].append(train_loss)
history['val_acc'].append(val_acc)
history['val_loss'].append(val_loss)
if val_acc > best_accuracy:
# here model.state_dict() will save the model and optimizer's parameter
torch.save(model.state_dict(), 'best_model.bin')
best_accuracy = val_acc
print ("Training completed...")
print ("Testing...\n")
# predictions
y_thesis, y_pred, y_pred_probs, y_test = predict.get_predictions(
model,
test_data_loader,
device
)
# Classification_report
print(classification_report(y_test, y_pred))
# Confusion matrix
ln = []
# tn, fp, fn, tp
ln = confusion_matrix(y_test, y_pred).ravel()
print ("True Negative, False Positive, False Negative, True positive : ", ln)
print ()
# Accuracy
print (accuracy_score(y_test, y_pred))
print()
print ("DONE!!")
my_trans = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
])
BATCH_SIZE = 16
MAX_LEN = 128
EPOCHS = 4
history = defaultdict(list)
best_accuracy = 0
LOAD_MODEL = False
train_data_loader = create_data_loader(train_df, tokenizer, MAX_LEN,
BATCH_SIZE, my_trans, 'Train', True)
val_data_loader = create_data_loader(val_df, tokenizer, MAX_LEN, BATCH_SIZE,
my_trans, 'Val', False)
model = multimodal()
model = model.to(device)
optimizer = AdamW(model.parameters(), lr=2e-5, correct_bias=False)
total_steps = len(train_data_loader) * EPOCHS
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps=0,
num_training_steps=total_steps)
loss = nn.BCEWithLogitsLoss().to(device)
for epoch in range(EPOCHS):