def run_alexnet(input_path, output_path, batch_size, epochs, learning_rate):
# Load image databunch
print("[INFO] Loading Data")
data = load_catsvsdog(input_path, batch_size)
# Defining the learner
alexnet_learner = Learner(
data=data,
model=ALEXNet(n_class=data.c),
loss_func=nn.CrossEntropyLoss(),
metrics=accuracy,
)
# Training the model
print("[INFO] Training started.")
alexnet_learner.fit_one_cycle(epochs, learning_rate)
# Validation accuracy
val_acc = int(
np.round(alexnet_learner.recorder.metrics[-1][0].numpy().tolist(), 3) *
1000)
# Saving the model
print("[INFO] Saving model weights.")
alexnet_learner.save("alexnet_catsvsdog_stg_1_" + str(val_acc))
# Evaluation
print("[INFO] Evaluating Network.")
evaluate_model(alexnet_learner, output_path, plot=True)
def run_mnist(input_path,
output_path,
batch_size,
epochs,
learning_rate,
model=Mnist_NN()):
path = Path(input_path)
## Defining transformation
ds_tfms = get_transforms(
do_flip=False,
flip_vert=False,
max_rotate=15,
max_zoom=1.1,
max_lighting=0.2,
max_warp=0.2,
)
## Creating Databunch
data = (ImageItemList.from_folder(path, convert_mode="L").split_by_folder(
train="training", valid="testing").label_from_folder().transform(
tfms=ds_tfms, size=28).databunch(bs=batch_size))
## Defining the learner
mlp_learner = Learner(data=data,
model=model,
loss_func=nn.CrossEntropyLoss(),
metrics=accuracy)
# Training the model
mlp_learner.fit_one_cycle(epochs, learning_rate)
val_acc = int(
np.round(mlp_learner.recorder.metrics[-1][0].numpy().tolist(), 3) *
1000)
## Saving the model
mlp_learner.save("mlp_mnist_stg_1_" + str(val_acc))
## Evaluation
print("Evaluating Network..")
interp = ClassificationInterpretation.from_learner(mlp_learner)
print(classification_report(interp.y_true, interp.pred_class))
## Plotting train and validation loss
mlp_learner.recorder.plot_losses()
plt.savefig(output_path + "/loss.png")
mlp_learner.recorder.plot_metrics()
plt.savefig(output_path + "/metric.png")
def run_shallownet(input_path, output_path, batch_size, epochs, learning_rate):
path = Path(input_path)
# Creating Databunch
data = (
ImageItemList.from_folder(path)
.split_by_folder(train="train", valid="test")
.label_from_folder()
.transform(tfms=None, size=32)
.databunch(bs=batch_size)
)
# Defining the learner
sn_learner = Learner(
data=data,
model=ShallowNet(n_class=data.c, size=32, in_channels=3),
loss_func=nn.CrossEntropyLoss(),
metrics=accuracy,
)
# Training the model
sn_learner.fit_one_cycle(epochs, learning_rate)
val_acc = int(
np.round(sn_learner.recorder.metrics[-1][0].numpy().tolist(), 3) * 1000
)
# Saving the model
sn_learner.save("sn_cifar10_stg_1_" + str(val_acc))
# Evaluation
print("Evaluating Network..")
interp = ClassificationInterpretation.from_learner(sn_learner)
print(classification_report(interp.y_true, interp.pred_class))
# Plotting train and validation loss
sn_learner.recorder.plot_losses()
plt.savefig(output_path + "/loss.png")
sn_learner.recorder.plot_metrics()
plt.savefig(output_path + "/metric.png")
def main(args):
if args.deterministic:
set_seed(42)
# Set device
if args.device is None:
if torch.cuda.is_available():
args.device = 'cuda:0'
else:
args.device = 'cpu'
defaults.device = torch.device(args.device)
# Aggregate path and labels into list for fastai ImageDataBunch
fnames, labels, is_valid = [], [], []
dataset = OpenFire(root=args.data_path,
train=True,
download=True,
img_folder=args.img_folder)
for sample in dataset.data:
fnames.append(
dataset._images.joinpath(sample['name']).relative_to(dataset.root))
labels.append(sample['target'])
is_valid.append(False)
dataset = OpenFire(root=args.data_path, train=False, download=True)
for sample in dataset.data:
fnames.append(
dataset._images.joinpath(sample['name']).relative_to(dataset.root))
labels.append(sample['target'])
is_valid.append(True)
df = pd.DataFrame.from_dict(
dict(name=fnames, label=labels, is_valid=is_valid))
# Split train and valid sets
il = vision.ImageList.from_df(
df, path=args.data_path).split_from_df('is_valid')
# Encode labels
il = il.label_from_df(cols='label',
label_cls=FloatList if args.binary else CategoryList)
# Set transformations
il = il.transform(vision.get_transforms(), size=args.resize)
# Create the Databunch
data = il.databunch(bs=args.batch_size,
num_workers=args.workers).normalize(
vision.imagenet_stats)
# Metric
metric = partial(vision.accuracy_thresh,
thresh=0.5) if args.binary else vision.error_rate
# Create model
model = models.__dict__[args.model](imagenet_pretrained=args.pretrained,
num_classes=data.c,
lin_features=args.lin_feats,
concat_pool=args.concat_pool,
bn_final=args.bn_final,
dropout_prob=args.dropout_prob)
# Create learner
learner = Learner(data,
model,
wd=args.weight_decay,
loss_func=CustomBCELogitsLoss()
if args.binary else nn.CrossEntropyLoss(),
metrics=metric)
# Form layer group for optimization
meta = model_meta.get(args.model, _default_meta)
learner.split(meta['split'])
# Freeze model's head
if args.pretrained:
learner.freeze()
if args.resume:
learner.load(args.resume)
if args.unfreeze:
learner.unfreeze()
learner.fit_one_cycle(args.epochs,
max_lr=slice(None, args.lr, None),
div_factor=args.div_factor,
final_div=args.final_div_factor)
learner.save(args.checkpoint)
num_workers=NUM_WORKERS,
normalization=STATISTICS,
)
# init model
swa_model = MODEL(num_classes=N_CLASSES, dropout_p=DROPOUT)
model = MODEL(num_classes=N_CLASSES, dropout_p=DROPOUT)
# nullify all swa model parameters
swa_params = swa_model.parameters()
for swa_param in swa_params:
swa_param.data = torch.zeros_like(swa_param.data)
# average model
n_swa = len(os.listdir(MODELS_FOLDER))
print(f"Averaging {n_swa} models")
for file in os.listdir(MODELS_FOLDER):
model.load_state_dict(torch.load(f'{MODELS_FOLDER}/{file}')['model'])
model_params = model.parameters()
for model_param, swa_param in zip(model_params, swa_params):
swa_param.data += model_param.data / n_swa
# fix batch norm
print("Fixing batch norm")
swa_model.to(DEVICE)
learn = Learner(data, model, model_dir=MODELS_FOLDER, loss_func=CRITERION, opt_func=OPTIMIZER, wd=WD)
learn.model = convert_model(learn.model)
learn.model = nn.DataParallel(learn.model).to(DEVICE)
fix_batchnorm(learn.model, learn.data.train_dl)
learn.save('swa_model')
def run_ner(
lang: str = 'eng',
log_dir: str = 'logs',
task: str = NER,
batch_size: int = 1,
lr: float = 5e-5,
epochs: int = 1,
dataset: str = 'data/conll-2003/',
loss: str = 'cross',
max_seq_len: int = 128,
do_lower_case: bool = False,
warmup_proportion: float = 0.1,
grad_acc_steps: int = 1,
rand_seed: int = None,
fp16: bool = False,
loss_scale: float = None,
ds_size: int = None,
data_bunch_path: str = 'data/conll-2003/db',
bertAdam: bool = False,
freez: bool = False,
one_cycle: bool = False,
discr: bool = False,
lrm: int = 2.6,
div: int = None,
tuned_learner: str = None,
do_train: str = False,
do_eval: str = False,
save: bool = False,
name: str = 'ner',
mask: tuple = ('s', 's'),
):
name = "_".join(
map(str, [
name, task, lang, mask[0], mask[1], loss, batch_size, lr,
max_seq_len, do_train, do_eval
]))
log_dir = Path(log_dir)
log_dir.mkdir(parents=True, exist_ok=True)
init_logger(log_dir, name)
if rand_seed:
random.seed(rand_seed)
np.random.seed(rand_seed)
torch.manual_seed(rand_seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(rand_seed)
trainset = dataset + lang + '/train.txt'
devset = dataset + lang + '/dev.txt'
testset = dataset + lang + '/test.txt'
bert_model = 'bert-base-cased' if lang == 'eng' else 'bert-base-multilingual-cased'
print(f'Lang: {lang}\nModel: {bert_model}\nRun: {name}')
model = BertForTokenClassification.from_pretrained(bert_model,
num_labels=len(VOCAB),
cache_dir='bertm')
model = torch.nn.DataParallel(model)
model_lr_group = bert_layer_list(model)
layers = len(model_lr_group)
kwargs = {'max_seq_len': max_seq_len, 'ds_size': ds_size, 'mask': mask}
train_dl = DataLoader(dataset=NerDataset(trainset,
bert_model,
train=True,
**kwargs),
batch_size=batch_size,
shuffle=True,
collate_fn=partial(pad, train=True))
dev_dl = DataLoader(dataset=NerDataset(devset, bert_model, **kwargs),
batch_size=batch_size,
shuffle=False,
collate_fn=pad)
test_dl = DataLoader(dataset=NerDataset(testset, bert_model, **kwargs),
batch_size=batch_size,
shuffle=False,
collate_fn=pad)
data = DataBunch(train_dl=train_dl,
valid_dl=dev_dl,
test_dl=test_dl,
collate_fn=pad,
path=Path(data_bunch_path))
loss_fun = ner_loss_func if loss == 'cross' else partial(ner_loss_func,
zero=True)
metrics = [Conll_F1()]
learn = Learner(
data,
model,
BertAdam,
loss_func=loss_fun,
metrics=metrics,
true_wd=False,
layer_groups=None if not freez else model_lr_group,
path='learn',
)
# initialise bert adam optimiser
train_opt_steps = int(len(train_dl.dataset) / batch_size) * epochs
optim = BertAdam(model.parameters(),
lr=lr,
warmup=warmup_proportion,
t_total=train_opt_steps)
if bertAdam: learn.opt = OptimWrapper(optim)
else: print("No Bert Adam")
# load fine-tuned learner
if tuned_learner:
print('Loading pretrained learner: ', tuned_learner)
learn.load(tuned_learner)
# Uncomment to graph learning rate plot
# learn.lr_find()
# learn.recorder.plot(skip_end=15)
# set lr (discriminative learning rates)
if div: layers = div
lrs = lr if not discr else learn.lr_range(slice(lr / lrm**(layers), lr))
results = [['epoch', 'lr', 'f1', 'val_loss', 'train_loss', 'train_losses']]
if do_train:
for epoch in range(epochs):
if freez:
lay = (layers // (epochs - 1)) * epoch * -1
if lay == 0:
print('Freeze')
learn.freeze()
elif lay == layers:
print('unfreeze')
learn.unfreeze()
else:
print('freeze2')
learn.freeze_to(lay)
print('Freezing layers ', lay, ' off ', layers)
# Fit Learner - eg train model
if one_cycle: learn.fit_one_cycle(1, lrs, moms=(0.8, 0.7))
else: learn.fit(1, lrs)
results.append([
epoch,
lrs,
learn.recorder.metrics[0][0],
learn.recorder.val_losses[0],
np.array(learn.recorder.losses).mean(),
learn.recorder.losses,
])
if save:
m_path = learn.save(f"{lang}_{epoch}_model", return_path=True)
print(f'Saved model to {m_path}')
if save: learn.export(f'{lang}.pkl')
if do_eval:
res = learn.validate(test_dl, metrics=metrics)
met_res = [f'{m.__name__}: {r}' for m, r in zip(metrics, res[1:])]
print(f'Validation on TEST SET:\nloss {res[0]}, {met_res}')
results.append(['val', '-', res[1], res[0], '-', '-'])
with open(log_dir / (name + '.csv'), 'a') as resultFile:
wr = csv.writer(resultFile)
wr.writerows(results)
