def run_ner_infer(sent):
device = 'cuda' if torch.cuda.is_available() else 'cpu'
top_rnns = True
model = Net(top_rnns, len(VOCAB), device, finetuning=True)
if device == 'cpu':
model.load_state_dict(
torch.load('models/banner_model.pt',
map_location=torch.device('cpu')))
elif device == 'cuda':
model.load_state_dict(torch.load('.models/banner_model.pt'))
model.to(device)
tags = []
for x in range(len(sent.split())):
tags.append('O')
sent_infer = []
sent_infer.append(["[CLS]"] + sent.split() + ["[SEP]"])
tags_infer = []
tags_infer.append(["<PAD>"] + tags + ["<PAD>"])
infer_data = NerDataset(sent_infer, tags_infer)
infer_iter = torch.utils.data.DataLoader(dataset=infer_data,
batch_size=1,
shuffle=False,
collate_fn=pad,
num_workers=0)
pred = eval(model, infer_iter)
for x in range(len(pred[0])):
if pred[0][x] == '<PAD>':
pred[0][x] = 'O'
return sent_infer[0][1:-1], pred[0][1:-1]
def main():
args = parse_args_finetuning_pruning()
print('------ Parameters for finetuning ------')
for parameter, value in args.__dict__.items():
print(f'{parameter}: {value}')
print('---------------------------------------')
if args.model_path is None:
if args.verbose:
print(f"No model was given, training {args.model} on {args.dataset} with {args.n_epochs} epochs.")
model = train_model(args)
else:
model = torch.load(args.model_path)
try:
os.mkdir("temp")
except FileExistsError:
pass
torch.save(model, "temp/model_finetuning_parameters.pt")
if not args.download and args.data_dir == '../data':
raise("ERROR: please provide the data directory from which to take the data.")
kwargs = {'num_workers': 1, 'pin_memory': True} if (torch.cuda.is_available() and args.use_cuda) else {}
device = torch.device("cuda:0" if (torch.cuda.is_available() and args.use_cuda) else "cpu")
loader_class = get_loader(args.dataset)
loader_object = loader_class(args.data_dir, args.batch_size, args.test_batch_size,
args.custom_transforms, args.crop_size)
loader_train = loader_object.get_loader(train=True, download=args.download, kwargs=kwargs)
loader_eval = loader_object.get_loader(train=False, download=args.download, kwargs=kwargs)
baseline_accuracy = eval(model, loader_eval, device, args.verbose)
accuracy_list = [baseline_accuracy]
n_epochs_retrain = args.n_epochs_retrain
for n_pruning_epochs in range(1, n_epochs_retrain + 1):
model_ = torch.load("temp/model_finetuning_parameters.pt")
accuracy_list.append(gradual_linear_pruning(model_, args.final_sparsity, loader_train, loader_eval,
n_epochs_retrain, n_pruning_epochs, 1, device, args.optimizer,
args.loss, args.lr, args.verbose, baseline_accuracy, args.save_to,
False, args.pruning_method))
if args.show_plot:
plt.plot(np.arange(n_epochs_retrain + 1), accuracy_list, label='Accuracy')
plt.xlabel('Pruning rate')
plt.ylabel('Accuracy')
plt.legend(loc="lower left")
plt.show()
def gradual_linear_pruning(model, final_sparsity, train_loader, test_loader,
n_epochs, pruning_epochs, frequency, device,
optimizer, loss, lr, verbose, acc_baseline,
save_path, show_plot, pruning_method):
if verbose:
print(
f"------Gradual linear pruning, global sparsity: {100*final_sparsity} % ------"
)
parameters_to_prune = [(module, 'weight')
for module in model.modules()][1:]
sparsity_list = np.zeros(n_epochs + 1)
accuracy_list = np.zeros(n_epochs + 1)
sparsity_list[0] = 0
accuracy_list[0] = acc_baseline
sparsity = 0
nb_of_pruning = 0
for epoch in range(1, n_epochs + 1):
if verbose:
print(f"Retraining: epoch {epoch} / {n_epochs}")
if nb_of_pruning < pruning_epochs and (epoch - 1) % frequency == 0:
nb_of_pruning += 1
new_sparsity = final_sparsity * nb_of_pruning / pruning_epochs
pruning_rate = (new_sparsity - sparsity) / (1 - sparsity)
prune_model(pruning_method, parameters_to_prune, pruning_rate)
sparsity = new_sparsity
train(model, train_loader, device, optimizer, loss, lr, verbose)
acc = eval(model, test_loader, device, verbose)
sparsity_list[epoch] = sparsity
accuracy_list[epoch] = acc
print("Global sparsity:", sparsity)
if save_path is not None:
torch.save(model, f"{save_path}_gradual_linear_pruning.pt")
if show_plot:
plt.plot(np.arange(n_epochs + 1), sparsity_list, label='Sparsity')
plt.plot(np.arange(n_epochs + 1), accuracy_list, label='Accuracy')
plt.title('Fine-tuning curve')
plt.xlabel('Epochs')
plt.ylabel('Sparsity / Accuracy')
plt.legend(loc="lower left")
plt.show()
return acc
def one_shot_pruning(model, final_sparsity, train_loader, test_loader,
n_epochs, device, optimizer, loss, lr, verbose,
acc_baseline, save_path, show_plot, pruning_method):
if verbose:
print(
f"------One shot pruning, global sparsity: {100*final_sparsity} % ------"
)
parameters_to_prune = [(module, 'weight')
for module in model.modules()][1:]
prune_model(pruning_method, parameters_to_prune, final_sparsity)
sparsity_list = np.zeros(n_epochs + 1)
accuracy_list = np.zeros(n_epochs + 1)
sparsity_list[0] = 0
accuracy_list[0] = acc_baseline
for epoch in range(1, n_epochs + 1):
if verbose:
print(f"Retraining: epoch {epoch} / {n_epochs}")
train(model, train_loader, device, optimizer, loss, lr, verbose)
acc = eval(model, test_loader, device, verbose)
sparsity_list[epoch] = final_sparsity
accuracy_list[epoch] = acc
if save_path is not None:
torch.save(model, f"{save_path}_one_shot_pruning.pt")
if show_plot:
plt.plot(np.arange(n_epochs + 1), sparsity_list, label='Sparsity')
plt.plot(np.arange(n_epochs + 1), accuracy_list, label='Accuracy')
plt.title('Fine-tuning curve')
plt.xlabel('Epochs')
plt.ylabel('Sparsity / Accuracy')
plt.legend(loc="lower left")
plt.show()
return acc
@author: ansh
"""
import torch
from ssdconfig import SSDConfig
from data import ShelfImageDataset, collate_fn, get_dataframe
from torch.utils.data import DataLoader
from trainer import eval
config = SSDConfig()
df = get_dataframe(config.PATH_TO_ANNOTATIONS)
dataset = ShelfImageDataset(df, config.PATH_TO_IMAGES, train=False)
dataloader = DataLoader(dataset,
shuffle=True,
collate_fn=collate_fn,
batch_size=config.TRAIN_BATCH_SIZE,
num_workers=config.NUM_DATALOADER_WORKERS)
ckpt_path = './checkpoints/checkpoint_ssd_1.pth.tar'
# ckpt_path = 'checkpoints/checkpoint_ssd_2-1AP.pth.tar'
checkpoint = torch.load(ckpt_path, map_location=torch.device('cpu'))
model = checkpoint['model']
model.config.DEVICE = torch.device('cpu')
mAP = eval(model, dataloader, min_score=0.5, max_overlap=0.5)
print(mAP)
collate_fn = pad,
num_workers=0
)
test_iter = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=batch_size,
shuffle=False,
collate_fn = pad,
num_workers=0
)
optimizer = optim.Adam(model.parameters(), lr = lr)
data_dist = [7237, 15684, 714867, 759, 20815, 9662, 8512, 37529, 70025]
crit_weights = crit_weights_gen(0.5,0.9,data_dist)
#insert 0 cost for ignoring <PAD>
crit_weights.insert(0,0)
crit_weights = torch.tensor(crit_weights).to(device)
criterion = nn.CrossEntropyLoss(weight=crit_weights)
for epoch in range(1, n_epochs+1):
if epoch>10:
optimizer = optim.Adam([
{"params": model.fc.parameters(), "lr": 0.0005},
{"params": model.bert.parameters(), "lr": 5e-5},
{"params": model.rnn.parameters(), "lr": 0.0005},
{"params": model.crf.parameters(), "lr": 0.0005}
],)
train(model, train_iter, optimizer, criterion, epoch)
_ = eval(model, test_iter, epoch)
fname = os.path.join("models", str(epoch))
torch.save(model.state_dict(), f"{fname}.pt")
def main():
args = parse_args_test_norm_pruning()
print('------ Parameters for test_norm_pruning ------')
for parameter, value in args.__dict__.items():
print(f'{parameter}: {value}')
print('------------------------------------------')
### Get the model, train it if none was given
if args.model_path is None:
model = train_model(args)
else:
model = torch.load(args.model_path)
### Save the trained model to make sure to have the same model before pruning.
try:
os.mkdir("temp")
except FileExistsError:
pass
torch.save(model, "temp/model_norm_pruning.pt")
### Get the loaders
if not args.download and args.data_dir == '../data':
raise (
"ERROR: please provide the data directory from which to take the data."
)
kwargs = {
'num_workers': 1,
'pin_memory': True
} if (torch.cuda.is_available() and args.use_cuda) else {}
device = torch.device("cuda:0" if (
torch.cuda.is_available() and args.use_cuda) else "cpu")
loader_class = get_loader(args.dataset)
loader_object = loader_class(args.data_dir, args.batch_size,
args.test_batch_size, args.custom_transforms,
args.crop_size)
loader_train = loader_object.get_loader(train=True,
download=args.download,
kwargs=kwargs)
loader_eval = loader_object.get_loader(train=False,
download=args.download,
kwargs=kwargs)
### Testing all the combination between the methods and pruning_rates given
pruning_rates = args.pruning_rates
methods = args.pruning_methods
baseline_accuracy = eval(model, loader_eval, device, args.verbose)
for method in methods:
accs = []
for pruning_rate in pruning_rates:
model_ = torch.load("temp/model_norm_pruning.pt")
accs.append(
one_shot_pruning(model_, pruning_rate, loader_train,
loader_eval, args.n_epochs_retrain, device,
args.optimizer, args.loss, args.lr,
args.verbose, baseline_accuracy, args.save_to,
args.show_plot, method))
plt.plot(pruning_rates, accs, label='Accuracy')
plt.title('Accuracy w.r.t pruning rate ' + method)
plt.xlabel('Pruning rate')
plt.ylabel('Accuracy')
plt.show()
def main():
args = parse_args_test_gradual_pruning()
print('------ Parameters for test_gradual_pruning ------')
for parameter, value in args.__dict__.items():
print(f'{parameter}: {value}')
print('------------------------------------------')
if args.model_path is None:
if args.verbose:
print(
f"No model was given, training {args.model} on {args.dataset} with {args.n_epochs} epochs."
)
model = train_model(args)
else:
model = torch.load(args.model_path)
try:
os.mkdir("temp")
except FileExistsError:
pass
torch.save(model, "temp/model_gradual_pruning.pt")
if not args.download and args.data_dir == '../data':
raise (
"ERROR: please provide the data directory from which to take the data."
)
kwargs = {
'num_workers': 1,
'pin_memory': True
} if (torch.cuda.is_available() and args.use_cuda) else {}
device = torch.device("cuda:0" if (
torch.cuda.is_available() and args.use_cuda) else "cpu")
loader_class = get_loader(args.dataset)
loader_object = loader_class(args.data_dir, args.batch_size,
args.test_batch_size, args.custom_transforms,
args.crop_size)
loader_train = loader_object.get_loader(train=True,
download=args.download,
kwargs=kwargs)
loader_eval = loader_object.get_loader(train=False,
download=args.download,
kwargs=kwargs)
baseline_accuracy = eval(model, loader_eval, device, args.verbose)
results_one_shot = one_shot_pruning(
model, args.final_sparsity, loader_train, loader_eval,
args.n_epochs_retrain, device, args.optimizer, args.loss, args.lr,
args.verbose, baseline_accuracy, args.save_to, args.show_plot,
args.pruning_method)
if args.verbose:
print(f"Accuracy obtained with one-shot pruning: {results_one_shot}")
results_linear_pruning = gradual_linear_pruning(
model, args.final_sparsity, loader_train, loader_eval,
args.n_epochs_retrain, args.pruning_epochs, args.pruning_frequency,
device, args.optimizer, args.loss, args.lr, args.verbose,
baseline_accuracy, args.save_to, args.show_plot, args.pruning_method)
if args.verbose:
print(
f"Accuracy obtained with linear gradual pruning: {results_one_shot}"
)
results_AGP = automated_gradual_pruning(
model, args.final_sparsity, loader_train, loader_eval,
args.n_epochs_retrain, args.pruning_epochs, args.pruning_frequency,
device, args.optimizer, args.loss, args.lr, args.verbose,
baseline_accuracy, args.save_to, args.show_plot, args.pruning_method)
if args.verbose:
print(
f"Accuracy obtained with automated gradual pruning: {results_AGP}")
def main(config):
# SET DEVICE
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(
str(gpu) for gpu in config["COMMON"]["GPUS"])
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
DATE = datetime.datetime.now().strftime("%Y_%m_%d/%H_%M_%S")
SAVEPATH = os.path.join(config["COMMON"]["SAVEPATH"], DATE)
config["COMMON"]["SAVEPATH"] = SAVEPATH
os.makedirs(SAVEPATH)
utils.set_logger(os.path.join(SAVEPATH, "train.log"))
utils.write_yaml(os.path.join(SAVEPATH, "config.yaml"), config)
# DATA LOADING
logging.info(f'Loading {config["DATA"]["NAME"]} datasets')
transform = [
transforms.RandomHorizontalFlip(),
transforms.RandomRotation(15)
]
loader = trainer.Dataloader(config["DATA"])
# check configuration & real
num_classes = len(loader["train"].dataset.classes)
assert num_classes == config["MODEL"][
"NUMCLASSES"], f'Number of class is not same!\nIn Directory: {num_classes}\nIn Configuration: {config["MODEL"]["NUMCLASSES"]}'
# PREPROCESSING
# Add New class
# Add New data
# MODEL BUILD
logging.info(f"Building model")
net = model.Model(config["MODEL"]["BASEMODEL"],
config["MODEL"]["NUMCLASSES"],
config["MODEL"]["FREEZE"]).to(device)
# net = model.Model(num_classes=config["MODEL"]["NUMCLASSES"]).to(device)
if torch.cuda.is_available() and len(config["COMMON"]["GPUS"]) > 1:
logging.info(f"Multi GPU mode")
net = torch.nn.DataParallel(
net, device_ids=config["COMMON"]["GPUS"]).to(device)
criterion = model.loss_fn
metrics = {"acc": model.accuracy} # If classification
# metrics = {}
optm = optm_dict[config["TRAIN"]["OPTIMIZER"]](
net.parameters(), lr=config["TRAIN"]["LEARNINGRATE"])
# TRAINING
EPOCHS = config["TRAIN"]["EPOCHS"]
logging.info(f"Training start !")
best_val_loss = np.inf
for epoch in range(EPOCHS):
metrics_summary = trainer.train(epoch, net, optm, criterion,
loader["train"], metrics, device,
config)
metrics_summary.update(
trainer.eval(epoch, net, optm, criterion, loader["validation"],
metrics, device, config))
metrics_string = " ; ".join(f"{key}: {value:05.3f}"
for key, value in metrics_summary.items())
logging.info(f"[{epoch+1}/{EPOCHS}] Performance: {metrics_string}")
is_best = metrics_summary['val_loss'] <= best_val_loss
utils.save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': net.state_dict(),
'optim_dict': optm.state_dict()
},
is_best=is_best,
checkpoint=SAVEPATH)
if is_best:
logging.info("Found new best loss !")
best_val_loss = metrics_summary['val_loss']
best_json_path = os.path.join(SAVEPATH, "metrics_best.json")
utils.save_dict_to_json(metrics_summary, best_json_path, is_best)
last_json_path = os.path.join(SAVEPATH, "metrics_history.json")
utils.save_dict_to_json(metrics_summary, last_json_path)
# Data version control
logging.info(f"Training done !")