def main():
if not os.path.isdir(CHECKPOINT):
os.makedirs(CHECKPOINT)
print('==> Preparing dataset')
trainloader, testloader = load_CIFAR(batch_size=BATCH_SIZE,
num_workers=NUM_WORKERS)
CLASSES = []
AUROCs = []
auroc = AverageMeter()
for t, cls in enumerate(ALL_CLASSES):
print('\nTask: [%d | %d]\n' % (t + 1, len(ALL_CLASSES)))
CLASSES = [cls]
print("==> Creating model")
model = LeNet(num_classes=1)
if CUDA:
model = model.cuda()
model = nn.DataParallel(model)
cudnn.benchmark = True
print(' Total params: %.2fK' %
(sum(p.numel() for p in model.parameters()) / 1000))
criterion = nn.BCELoss()
optimizer = optim.SGD(model.parameters(),
lr=LEARNING_RATE,
momentum=MOMENTUM,
weight_decay=WEIGHT_DECAY)
print("==> Learning")
best_loss = 1e10
learning_rate = LEARNING_RATE
for epoch in range(EPOCHS):
# decay learning rate
if (epoch + 1) % EPOCHS_DROP == 0:
learning_rate *= LR_DROP
for param_group in optimizer.param_groups:
param_group['lr'] = learning_rate
print('Epoch: [%d | %d]' % (epoch + 1, EPOCHS))
train_loss = train(trainloader,
model,
criterion,
CLASSES,
CLASSES,
optimizer=optimizer,
use_cuda=CUDA)
test_loss = train(testloader,
model,
criterion,
CLASSES,
CLASSES,
test=True,
use_cuda=CUDA)
# save model
is_best = test_loss < best_loss
best_loss = min(test_loss, best_loss)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'loss': test_loss,
'optimizer': optimizer.state_dict()
}, CHECKPOINT, is_best)
print("==> Calculating AUROC")
filepath_best = os.path.join(CHECKPOINT, "best.pt")
checkpoint = torch.load(filepath_best)
model.load_state_dict(checkpoint['state_dict'])
new_auroc = calc_avg_AUROC(model, testloader, CLASSES, CLASSES, CUDA)
auroc.update(new_auroc)
print('New Task AUROC: {}'.format(new_auroc))
print('Average AUROC: {}'.format(auroc.avg))
AUROCs.append(auroc.avg)
print('\nAverage Per-task Performance over number of tasks')
for i, p in enumerate(AUROCs):
print("%d: %f" % (i + 1, p))
def main():
use_cuda = torch.cuda.is_available() and not args.no_cuda
device = torch.device('cuda' if use_cuda else 'cpu')
print(device)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if use_cuda:
torch.cuda.manual_seed(args.seed)
torch.backends.cudnn.deterministic = True
rgb = False
if args.mode == 'rgb':
rgb = True
if args.gray_scale:
rgb = False
if args.tracking_data_mod is True:
args.input_size = 192
# DATALOADER
train_dataset = GesturesDataset(model=args.model, csv_path='csv_dataset', train=True, mode=args.mode, rgb=rgb,
normalization_type=1,
n_frames=args.n_frames, resize_dim=args.input_size,
transform_train=args.train_transforms, tracking_data_mod=args.tracking_data_mod)
train_loader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.n_workers)
validation_dataset = GesturesDataset(model=args.model, csv_path='csv_dataset', train=False, mode=args.mode, rgb=rgb, normalization_type=1,
n_frames=args.n_frames, resize_dim=args.input_size, tracking_data_mod=args.tracking_data_mod)
validation_loader = DataLoader(validation_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.n_workers)
# paramteri per la rete
in_channels = args.n_frames if not rgb else args.n_frames * 3
n_classes = args.n_classes
if args.model == 'LeNet':
model = LeNet(input_channels=in_channels, input_size=args.input_size, n_classes=n_classes).to(device)
elif args.model == 'AlexNet':
model = AlexNet(input_channels=in_channels, input_size=args.input_size, n_classes=n_classes).to(device)
elif args.model == 'AlexNetBN':
model = AlexNetBN(input_channels=in_channels, input_size=args.input_size, n_classes=n_classes).to(device)
elif args.model == "Vgg16":
model = Vgg16(input_channels=in_channels, input_size=args.input_size, n_classes=n_classes).to(device)
elif args.model == "Vgg16P":
model = models.vgg16(pretrained=args.pretrained)
for params in model.parameters():
params.requires_grad = False
model.features._modules['0'] = nn.Conv2d(in_channels=in_channels, out_channels=64, kernel_size=(3, 3), stride=1, padding=1)
model.classifier._modules['6'] = nn.Linear(4096, n_classes)
# model.fc = torch.nn.Linear(model.fc.in_features, n_classes)
model = model.to(device)
elif args.model == "ResNet18P":
model = models.resnet18(pretrained=args.pretrained)
for params in model.parameters():
params.requires_grad = False
model._modules['conv1'] = nn.Conv2d(in_channels, 64, 7, stride=2, padding=3)
model.fc = torch.nn.Linear(model.fc.in_features, n_classes)
model = model.to(device)
elif args.model == "ResNet34P":
model = models.resnet34(pretrained=args.pretrained)
for params in model.parameters():
params.requires_grad = False
model._modules['conv1'] = nn.Conv2d(in_channels, 64, 7, stride=2, padding=3)
model.fc = torch.nn.Linear(model.fc.in_features, n_classes)
model = model.to(device)
elif args.model == "DenseNet121P":
model = models.densenet121(pretrained=args.pretrained)
for params in model.parameters():
params.requires_grad = False
model.features._modules['conv0'] = nn.Conv2d(in_channels=in_channels, out_channels=64, kernel_size=(7, 7),
stride=(2, 2), padding=(3, 3))
model.classifier = nn.Linear(in_features=1024, out_features=n_classes, bias=True)
model = model.to(device)
elif args.model == "DenseNet161P":
model = models.densenet161(pretrained=args.pretrained)
# for params in model.parameters():
# params.requires_grad = False
model.features._modules['conv0'] = nn.Conv2d(in_channels=in_channels, out_channels=96, kernel_size=(7, 7),
stride=(2, 2), padding=(3, 3))
model.classifier = nn.Linear(in_features=2208, out_features=n_classes, bias=True)
model = model.to(device)
elif args.model == "DenseNet169P":
model = models.densenet169(pretrained=args.pretrained)
for params in model.parameters():
params.requires_grad = False
model.features._modules['conv0'] = nn.Conv2d(in_channels=in_channels, out_channels=64, kernel_size=(7, 7),
stride=(2, 2), padding=(3, 3))
model.classifier = nn.Linear(in_features=1664, out_features=n_classes, bias=True)
model = model.to(device)
elif args.model == "DenseNet201P":
model = models.densenet201(pretrained=args.pretrained)
for params in model.parameters():
params.requires_grad = False
model.features._modules['conv0'] = nn.Conv2d(in_channels=in_channels, out_channels=64, kernel_size=(7, 7),
stride=(2, 2), padding=(3, 3))
model.classifier = nn.Linear(in_features=1920, out_features=n_classes, bias=True)
model = model.to(device)
# RNN
elif args.model == 'LSTM' or args.model == 'GRU':
model = Rnn(rnn_type=args.model, input_size=args.input_size, hidden_size=args.hidden_size,
batch_size=args.batch_size,
num_classes=args.n_classes, num_layers=args.n_layers,
final_layer=args.final_layer).to(device)
# C3D
elif args.model == 'C3D':
if args.pretrained:
model = C3D(rgb=rgb, num_classes=args.n_classes)
# modifico parametri
print('ok')
model.load_state_dict(torch.load('c3d_weights/c3d.pickle', map_location=device), strict=False)
# # for params in model.parameters():
# # params.requires_grad = False
model.conv1 = nn.Conv3d(1 if not rgb else 3, 64, kernel_size=(3, 3, 3), padding=(1, 1, 1))
# tolgo fc6 perchè 30 frames
model.fc6 = nn.Linear(16384, 4096) # num classes 28672 (112*200)
model.fc7 = nn.Linear(4096, 4096) # num classes
model.fc8 = nn.Linear(4096, n_classes) # num classes
model = model.to(device)
# Conv-lstm
elif args.model == 'Conv-lstm':
model = ConvLSTM(input_size=(args.input_size, args.input_size),
input_dim=1 if not rgb else 3,
hidden_dim=[64, 64, 128],
kernel_size=(3, 3),
num_layers=args.n_layers,
batch_first=True,
).to(device)
elif args.model == 'DeepConvLstm':
model = DeepConvLstm(input_channels_conv=1 if not rgb else 3, input_size_conv=args.input_size, n_classes=12,
n_frames=args.n_frames, batch_size=args.batch_size).to(device)
elif args.model == 'ConvGRU':
model = ConvGRU(input_size=40, hidden_sizes=[64, 128],
kernel_sizes=[3, 3], n_layers=2).to(device)
else:
raise NotImplementedError
if args.opt == 'SGD':
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)
# optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr, momentum=args.momentum)
elif args.opt == 'Adam':
optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
loss_function = nn.CrossEntropyLoss().to(device)
start_epoch = 0
if args.resume:
checkpoint = torch.load("/projects/fabio/weights/gesture_recog_weights/checkpoint{}.pth.tar".format(args.model))
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
start_epoch = checkpoint['epoch']
print("Resuming state:\n-epoch: {}\n{}".format(start_epoch, model))
#name experiment
personal_name = "{}_{}_{}".format(args.model, args.mode, args.exp_name)
info_experiment = "{}".format(personal_name)
log_dir = "/projects/fabio/logs/gesture_recog_logs/exps"
weight_dir = personal_name
log_file = open("{}/{}.txt".format("/projects/fabio/logs/gesture_recog_logs/txt_logs", personal_name), 'w')
log_file.write(personal_name + "\n\n")
if personal_name:
exp_name = (("exp_{}_{}".format(time.strftime("%c"), personal_name)).replace(" ", "_")).replace(":", "-")
else:
exp_name = (("exp_{}".format(time.strftime("%c"), personal_name)).replace(" ", "_")).replace(":", "-")
writer = SummaryWriter("{}".format(os.path.join(log_dir, exp_name)))
# add info experiment
writer.add_text('Info experiment',
"model:{}"
"\n\npretrained:{}"
"\n\nbatch_size:{}"
"\n\nepochs:{}"
"\n\noptimizer:{}"
"\n\nlr:{}"
"\n\ndn_lr:{}"
"\n\nmomentum:{}"
"\n\nweight_decay:{}"
"\n\nn_frames:{}"
"\n\ninput_size:{}"
"\n\nhidden_size:{}"
"\n\ntracking_data_mode:{}"
"\n\nn_classes:{}"
"\n\nmode:{}"
"\n\nn_workers:{}"
"\n\nseed:{}"
"\n\ninfo:{}"
"".format(args.model, args.pretrained, args.batch_size, args.epochs, args.opt, args.lr, args.dn_lr, args.momentum,
args.weight_decay, args.n_frames, args.input_size, args.hidden_size, args.tracking_data_mod,
args.n_classes, args.mode, args.n_workers, args.seed, info_experiment))
trainer = Trainer(model=model, loss_function=loss_function, optimizer=optimizer, train_loader=train_loader,
validation_loader=validation_loader,
batch_size=args.batch_size, initial_lr=args.lr, device=device, writer=writer, personal_name=personal_name, log_file=log_file,
weight_dir=weight_dir, dynamic_lr=args.dn_lr)
print("experiment: {}".format(personal_name))
start = time.time()
for ep in range(start_epoch, args.epochs):
trainer.train(ep)
trainer.val(ep)
# display classes results
classes = ['g0', 'g1', 'g2', 'g3', 'g4', 'g5', 'g6', 'g7', 'g8', 'g9', 'g10', 'g11']
for i in range(args.n_classes):
print('Accuracy of {} : {:.3f}%%'.format(
classes[i], 100 * trainer.class_correct[i] / trainer.class_total[i]))
end = time.time()
h, rem = divmod(end - start, 3600)
m, s, = divmod(rem, 60)
print("\nelapsed time (ep.{}):{:0>2}:{:0>2}:{:05.2f}".format(args.epochs, int(h), int(m), s))
# writing accuracy on file
log_file.write("\n\n")
for i in range(args.n_classes):
log_file.write('Accuracy of {} : {:.3f}%\n'.format(
classes[i], 100 * trainer.class_correct[i] / trainer.class_total[i]))
log_file.close()
def main():
# Data Loader (Input Pipeline)
print('loading dataset...')
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=batch_size,
num_workers=args.num_workers,
drop_last=False,
shuffle=False)
val_loader = torch.utils.data.DataLoader(dataset=val_dataset,
batch_size=batch_size,
num_workers=args.num_workers,
drop_last=False,
shuffle=False)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=batch_size,
num_workers=args.num_workers,
drop_last=False,
shuffle=False)
# Define models
print('building model...')
if args.dataset == 'mnist':
clf1 = LeNet()
if args.dataset == 'fashionmnist':
clf1 = resnet.ResNet18_F(10)
if args.dataset == 'cifar10':
clf1 = resnet.ResNet34(10)
if args.dataset == 'svhn':
clf1 = resnet.ResNet34(10)
clf1.cuda()
optimizer = torch.optim.SGD(clf1.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
with open(txtfile, "a") as myfile:
myfile.write('epoch train_acc val_acc test_acc\n')
epoch = 0
train_acc = 0
val_acc = 0
# evaluate models with random weights
test_acc = evaluate(test_loader, clf1)
print('Epoch [%d/%d] Test Accuracy on the %s test data: Model1 %.4f %%' %
(epoch + 1, args.n_epoch_1, len(test_dataset), test_acc))
# save results
with open(txtfile, "a") as myfile:
myfile.write(
str(int(epoch)) + ' ' + str(train_acc) + ' ' + str(val_acc) + ' ' +
str(test_acc) + ' ' + "\n")
best_acc = 0.0
# training
for epoch in range(1, args.n_epoch_1):
# train models
clf1.train()
train_acc = train(clf1, train_loader, epoch, optimizer,
nn.CrossEntropyLoss())
# validation
val_acc = evaluate(val_loader, clf1)
# evaluate models
test_acc = evaluate(test_loader, clf1)
# save results
print(
'Epoch [%d/%d] Train Accuracy on the %s train data: Model %.4f %%'
% (epoch + 1, args.n_epoch_1, len(train_dataset), train_acc))
print('Epoch [%d/%d] Val Accuracy on the %s val data: Model %.4f %% ' %
(epoch + 1, args.n_epoch_1, len(val_dataset), val_acc))
print(
'Epoch [%d/%d] Test Accuracy on the %s test data: Model %.4f %% ' %
(epoch + 1, args.n_epoch_1, len(test_dataset), test_acc))
with open(txtfile, "a") as myfile:
myfile.write(
str(int(epoch)) + ' ' + str(train_acc) + ' ' + str(val_acc) +
' ' + str(test_acc) + ' ' + "\n")
if val_acc > best_acc:
best_acc = val_acc
torch.save(clf1.state_dict(), model_save_dir + '/' + 'model.pth')
print('Matrix Factorization is doing...')
clf1.load_state_dict(torch.load(model_save_dir + '/' + 'model.pth'))
A = respresentations_extract(train_loader, clf1, len(train_dataset),
args.dim, batch_size)
A_val = respresentations_extract(val_loader, clf1, len(val_dataset),
args.dim, batch_size)
A_total = np.append(A, A_val, axis=0)
W_total, H_total, error = train_m(A_total, args.basis, args.iteration_nmf,
1e-5)
for i in range(W_total.shape[0]):
for j in range(W_total.shape[1]):
if W_total[i, j] < 1e-6:
W_total[i, j] = 0.
W = W_total[0:len(train_dataset), :]
W_val = W_total[len(train_dataset):, :]
print('Transition Matrix is estimating...Wating...')
logits_matrix = probability_extract(train_loader, clf1, len(train_dataset),
args.num_classes, batch_size)
idx_matrix_group, transition_matrix_group = estimate_matrix(
logits_matrix, model_save_dir)
logits_matrix_val = probability_extract(val_loader, clf1, len(val_dataset),
args.num_classes, batch_size)
idx_matrix_group_val, transition_matrix_group_val = estimate_matrix(
logits_matrix_val, model_save_dir)
func = nn.MSELoss()
model = Matrix_optimize(args.basis, args.num_classes)
optimizer_1 = torch.optim.Adam(model.parameters(), lr=0.001)
basis_matrix_group = basis_matrix_optimize(model, optimizer_1, args.basis,
args.num_classes, W,
transition_matrix_group,
idx_matrix_group, func,
model_save_dir, args.n_epoch_4)
basis_matrix_group_val = basis_matrix_optimize(
model, optimizer_1, args.basis, args.num_classes, W_val,
transition_matrix_group_val, idx_matrix_group_val, func,
model_save_dir, args.n_epoch_4)
for i in range(basis_matrix_group.shape[0]):
for j in range(basis_matrix_group.shape[1]):
for k in range(basis_matrix_group.shape[2]):
if basis_matrix_group[i, j, k] < 1e-6:
basis_matrix_group[i, j, k] = 0.
optimizer_ = torch.optim.SGD(clf1.parameters(),
lr=args.lr,
weight_decay=args.weight_decay,
momentum=args.momentum)
best_acc = 0.0
for epoch in range(1, args.n_epoch_2):
# train model
clf1.train()
train_acc = train_correction(clf1, train_loader, epoch, optimizer_, W,
basis_matrix_group, batch_size,
args.num_classes, args.basis)
# validation
val_acc = val_correction(clf1, val_loader, epoch, W_val,
basis_matrix_group_val, batch_size,
args.num_classes, args.basis)
# evaluate models
test_acc = evaluate(test_loader, clf1)
if val_acc > best_acc:
best_acc = val_acc
torch.save(clf1.state_dict(), model_save_dir + '/' + 'model.pth')
with open(txtfile, "a") as myfile:
myfile.write(
str(int(epoch)) + ' ' + str(train_acc) + ' ' + str(val_acc) +
' ' + str(test_acc) + ' ' + "\n")
# save results
print(
'Epoch [%d/%d] Train Accuracy on the %s train data: Model %.4f %%'
% (epoch + 1, args.n_epoch_2, len(train_dataset), train_acc))
print('Epoch [%d/%d] Val Accuracy on the %s val data: Model %.4f %% ' %
(epoch + 1, args.n_epoch_2, len(val_dataset), val_acc))
print(
'Epoch [%d/%d] Test Accuracy on the %s test data: Model %.4f %% ' %
(epoch + 1, args.n_epoch_2, len(test_dataset), test_acc))
clf1.load_state_dict(torch.load(model_save_dir + '/' + 'model.pth'))
optimizer_r = torch.optim.Adam(clf1.parameters(),
lr=args.lr_revision,
weight_decay=args.weight_decay)
nn.init.constant_(clf1.T_revision.weight, 0.0)
for epoch in range(1, args.n_epoch_3):
# train models
clf1.train()
train_acc = train_revision(clf1, train_loader, epoch, optimizer_r, W,
basis_matrix_group, batch_size,
args.num_classes, args.basis)
# validation
val_acc = val_revision(clf1, val_loader, epoch, W_val,
basis_matrix_group, batch_size,
args.num_classes, args.basis)
# evaluate models
test_acc = evaluate(test_loader, clf1)
with open(txtfile, "a") as myfile:
myfile.write(
str(int(epoch)) + ' ' + str(train_acc) + ' ' + str(val_acc) +
' ' + str(test_acc) + ' ' + "\n")
# save results
print(
'Epoch [%d/%d] Train Accuracy on the %s train data: Model %.4f %%'
% (epoch + 1, args.n_epoch_3, len(train_dataset), train_acc))
print('Epoch [%d/%d] Val Accuracy on the %s val data: Model %.4f %% ' %
(epoch + 1, args.n_epoch_3, len(val_dataset), val_acc))
print(
'Epoch [%d/%d] Test Accuracy on the %s test data: Model %.4f %% ' %
(epoch + 1, args.n_epoch_3, len(test_dataset), test_acc))
pin_memory=True)
net = LeNet()
criterion = nn.CrossEntropyLoss()
if use_cuda:
print('start move to cuda')
torch.cuda.manual_seed_all(seed)
cudnn.benchmark = True
device = torch.device("cuda:0")
net.to(device=device)
criterion.to(device=device, dtype=dtype)
optimizer = optim.Adam(
net.parameters(),
lr=0.001,
)
def train(epoch):
# global monitor
print('\nEpoch: %d' % epoch)
net.train()
train_loss = 0
correct = 0
a = time.time()
for batch_idx, (data, target) in enumerate(train_loader):
if use_cuda:
data, target = data.to(device=device,
if test_type == 'single task classification':
root = './data'
download = True # download MNIST dataset or not
trans = transforms.Compose(
[transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
train_set = dset.MNIST(root=root, train=True,
transform=trans, download=download)
test_set = dset.MNIST(root=root, train=False, transform=trans)
model = LeNet()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
model.compile(optimizer, criterion, metrics=['top1', 'top2'])
elif test_type == 'multi tasks classification':
train_set = CategoricalDatasetMultiTasks(N, D_in, D_out1, D_out2)
test_set = CategoricalDatasetMultiTasks(
int(N * 0.25), D_in, D_out1, D_out2)
model = MultiTasksClassification(D_in, H1, H2, D_out1, D_out2)
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
model.compile(optimizer, multiCrossEntropy, metrics=[
'top1', 'top2'], multi_tasks=['output_a', 'output_b'])
img = data
img = img.numpy()
img = np.transpose(img, (1, 2, 0))
plt.imshow(img, cmap='gray')
plt.title('%s' % train_dataset.classes[label])
plt.show()
net = LeNet()
print(net)
if os.path.isfile('saves/dogcat_lenet_params.pkl'):
net.load_state_dict(torch.load('saves/dogcat_lenet_params.pkl'))
# optimizer & Loss
optimizer = torch.optim.Adam(net.parameters(), lr=LR)
loss_func = nn.CrossEntropyLoss()
for epoch in range(EPOCH):
for step, (images, labels) in enumerate(train_loader):
images = Variable(images)
labels = Variable(labels)
optimizer.zero_grad()
prediction = net(images)
loss = loss_func(prediction, labels)
loss.backward()
optimizer.step()
# Model
print('==> Building model..')
model = LeNet(3, len(classes)).to(device)
if device.type == 'cuda':
model = torch.nn.DataParallel(model)
cudnn.benchmark = True
if os.path.isfile(ckpt_file):
checkpoint = torch.load(ckpt_file)
model.load_state_dict(checkpoint['net'])
best_acc = checkpoint['acc']
start_epoch = checkpoint['epoch']
criterion = nn.NLLLoss()
optimizer = optim.SGD(model.parameters(),
lr=1e-3,
momentum=0.9,
weight_decay=5e-4)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')
# Training
def train(epoch):
print('\nEpoch: %d' % epoch)
model.train()
train_loss = []
correct = 0
total = 0
for batch_idx, (inputs, targets) in enumerate(trainloader):
inputs, targets = inputs.to(device), targets.to(device)
def main():
opt = Parser().get()
experiment = None
if opt.comet:
experiment = Experiment()
experiment.set_name(opt.exp_name)
experiment.log_parameters(opt.__dict__)
experiment.add_tags(opt.add_tags)
# dataset and data loader
train_loader, val_loader, adv_val_loader, _, num_classes = \
load_dataset(opt.dataset, opt.batch_size, opt.data_root,
opt.noise, opt.noise_std, opt.num_val_samples,
workers=4)
# model
if opt.arch == 'lenet':
model = LeNet(num_classes)
elif opt.arch == 'resnet':
model = ResNetv2_20(num_classes)
else:
raise NotImplementedError
# weight init
if opt.weight_init == 'he':
model.apply(init_he)
# move model to device
model.to(opt.device)
if opt.gpu_ids:
model = nn.DataParallel(model, device_ids=opt.gpu_ids)
# criterion
criterion = nn.CrossEntropyLoss()
# advertorch attacker
if opt.attack == 'pgd':
attacker = LinfPGDAttack(model,
loss_fn=criterion,
eps=opt.eps / 255,
nb_iter=opt.num_steps,
eps_iter=opt.eps_iter / 255,
rand_init=True,
clip_min=opt.clip_min,
clip_max=opt.clip_max,
targeted=False)
else:
raise NotImplementedError
# optimizer
if opt.optim == 'Adam':
optimizer = optim.Adam(model.parameters(),
opt.lr,
eps=1e-6,
weight_decay=opt.wd)
elif opt.optim == 'SGD':
optimizer = optim.SGD(model.parameters(), opt.lr, weight_decay=opt.wd)
else:
raise NotImplementedError
# scheduler
if opt.scheduler_step:
scheduler = optim.lr_scheduler.StepLR(optimizer,
step_size=opt.scheduler_step,
gamma=opt.scheduler_gamma)
else:
scheduler = None
# timer
timer = Timer(opt.num_epochs, 0)
# trainer
trainer = Trainer(opt, model, criterion, attacker, optimizer)
# epoch iteration
for epoch in range(1, opt.num_epochs + 1):
trainer.epoch = epoch
if scheduler:
scheduler.step(epoch - 1) # scheduler's epoch is 0-indexed.
# training
train_losses, train_acc1s, train_acc5s = \
trainer.train(train_loader)
# validation
val_losses, val_acc1s, val_acc5s = \
trainer.validate(val_loader)
if opt.adv_val_freq != -1 and epoch % opt.adv_val_freq == 0:
aval_losses, aval_acc1s, aval_acc5s = \
trainer.adv_validate(adv_val_loader)
else:
aval_losses, aval_acc1s, aval_acc5s = \
dict(), dict(), dict()
losses = dict(**train_losses, **val_losses, **aval_losses)
acc1s = dict(**train_acc1s, **val_acc1s, **aval_acc1s)
acc5s = dict(**train_acc5s, **val_acc5s, **aval_acc5s)
report_epoch_status(losses, acc1s, acc5s, trainer.num_loss, epoch, opt,
timer, experiment)
save_path = os.path.join('ckpt', opt.dataset, 'models',
opt.exp_name + '.pth')
trainer.save_model(save_path)
SH.perf.first_epoch = []
# Store test performance after each args.save_interval iterations
SH.perf.te_vs_iterations = []
# Create model, and expose modules that has_codes
if algName == 'altmin':
model = get_mods(model)
set_mod_optimizers_(model,
optimizer='Adam',
optimizer_params={'lr': args.lr_weights},
data_parallel=multi_gpu)
model[-1].optimizer.param_groups[0]['lr'] = args.lr_weights
elif algName == 'adam':
optimizer = optim.Adam(model.parameters(), lr=args.lr_weights)
elif algName == 'sgd':
optimizer = optim.SGD(model.parameters(), lr=args.lr_weights)
scheduler = optim.lr_scheduler.StepLR(optimizer,
step_size=1,
gamma=0.9)
# Initial mu and increment after every mini-batch
mu = args.min_mu
mu_max = args.max_mu
for epoch in range(1, args.epochs + 1):
# Configure dropout parameters.
if args.LW_dropout_perc > 0 and args.LW_dropout_delay < epoch:
useDropout = args.LW_dropout_perc
def main():
# Training settings
parser = argparse.ArgumentParser(description='Adversarial MNIST Example')
parser.add_argument('--use-pretrained',
action='store_true',
default=False,
help='uses the pretrained model')
parser.add_argument('--adversarial-training',
action='store_true',
default=False,
help='takes the adversarial training process')
parser.add_argument('--batch-size',
type=int,
default=64,
metavar='N',
help='input batch size for training (default: 64)')
parser.add_argument('--test-batch-size',
type=int,
default=512,
metavar='N',
help='input batch size for testing (default: 512)')
args = parser.parse_args()
# Define what device we are using
use_cuda = torch.cuda.is_available()
print("CUDA Available: ", use_cuda)
device = torch.device("cuda" if use_cuda else "cpu")
# MNIST Test dataset and dataloader declaration
kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
train_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../data',
train=True,
download=True,
transform=transforms.Compose([transforms.ToTensor()])),
batch_size=args.batch_size,
shuffle=True,
**kwargs)
test_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../data',
train=False,
transform=transforms.Compose([transforms.ToTensor()])),
batch_size=args.test_batch_size,
shuffle=True,
**kwargs)
# Initialize the network
model = LeNet().to(device)
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5)
if args.use_pretrained:
print('Loading the pretrained model')
model.load_state_dict(
torch.load('resources/lenet_mnist_model.bin', map_location='cpu'))
else:
print('Training on the MNIST dataset')
model_train(model, train_loader, F.nll_loss, optimizer, epochs=10)
print('Evaluating the neural network')
# Evaluate the accuracy of the MNIST model on clean examples
accuracy, _ = model_eval(model, test_loader, F.nll_loss)
print('Test accuracy on clean examples: ' + str(accuracy))
# Evaluate the accuracy of the MNIST model on adversarial examples
accuracy, _ = model_eval(model,
test_loader,
F.nll_loss,
attack_method=fgsm_attack)
print('Test accuracy on adversarial examples: ' + str(accuracy))
if args.adversarial_training:
print("Repeating the process, with adversarial training")
# Perform adversarial training
model_train(model,
train_loader,
F.nll_loss,
optimizer,
epochs=10,
attack_method=fgsm_attack)
# Evaluate the accuracy of the adversarially trained MNIST model on
# clean examples
accuracy, _ = model_eval(model, test_loader, F.nll_loss)
print('Test accuracy on clean examples: ' + str(accuracy))
# Evaluate the accuracy of the adversarially trained MNIST model on
# adversarial examples
accuracy_adv, _ = model_eval(model,
test_loader,
F.nll_loss,
attack_method=fgsm_attack)
print('Test accuracy on adversarial examples: ' + str(accuracy_adv))
train=True,
transform=train_transform,
download=True)
fmnist_test = datasets.FashionMNIST(FMNIST_DATA_HOME,
train=False,
transform=test_transform,
download=True)
m = 5000
train_data = Subset(fmnist_train, m)
val_data = Subset(fmnist_train, np.arange(m, m + 200))
val_data2 = Subset(fmnist_train, np.arange(m - 200, m))
test_data = fmnist_test
net = LeNet(in_channels=1)
optimizer = SGD(net.parameters(), lr=0.003, momentum=0.9, weight_decay=5e-4)
batch_size = 64
data_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)
epochs = 100
for epoch in range(epochs):
if epoch % 2 == 0:
gc.collect()
print("Epoch %d" % (epoch + 1))
for i, batch in enumerate(data_loader):
inputs, labels = batch
inputs = H.tensor(inputs.numpy())
labels = H.tensor(labels.numpy())
net.zero_grad()
output = net(inputs)
model = LeNet()
class CELog(nn.Module):
def __init__(self):
super().__init__()
self.nl = nn.NLLLoss()
def forward(self, x, y):
return self.nl(x.log(), y)
criterion = CELog()
## Choose optimizer to be PolyakSGD
optimizer = PolyakSGD(model.parameters(),
max_lr=0.1,
fstar=0.0,
window=20 * int(60000 / train_batch_size))
##################################################
## Train and test the model over several epochs ##
##################################################
## Define the function to train the model
def train(e):
model.train()
for p in model.parameters():
p.requires_grad_(True)
class Reptile(object):
def __init__(self, args):
self.args = args
self._load_model()
self.model.to(args.device)
self.task_generator = TaskGen(args.max_num_classes)
self.outer_stepsize = args.outer_stepsize
self.criterion = nn.CrossEntropyLoss()
# self.optimizer = optim.Adam(self.model.parameters(), lr=args.inner_stepsize)
def _load_model(self):
self.model = LeNet()
self.current_iteration = 0
if os.path.exists(self.args.model_path):
try:
print("Loading model from: {}".format(self.args.model_path))
self.model.load_state_dict(torch.load(self.args.model_path))
self.current_iteration = joblib.load("{}.iter".format(
self.args.model_path))
except Exception as e:
print(
"Exception: {}\nCould not load model from {} - starting from scratch"
.format(e, self.args.model_path))
def inner_training(self, x, y, num_iterations):
"""
Run training on task
"""
x, y = shuffle_unison(x, y)
self.model.train()
x = torch.tensor(x, dtype=torch.float, device=self.args.device)
y = torch.tensor(y, dtype=torch.float, device=self.args.device)
total_loss = 0
for _ in range(num_iterations):
start = np.random.randint(0,
len(x) - self.args.inner_batch_size + 1)
self.model.zero_grad()
# self.optimizer.zero_grad()
outputs = self.model(x[start:start + self.args.inner_batch_size])
# print("output: {} - y: {}".format(outputs.shape, y.shape))
loss = self.criterion(
outputs,
Variable(y[start:start + self.args.inner_batch_size].long()))
total_loss += loss
loss.backward()
# self.optimizer.step()
# Similar to calling optimizer.step()
for param in self.model.parameters():
param.data -= self.args.inner_stepsize * param.grad.data
return total_loss / self.args.inner_iterations
def _meta_gradient_update(self, iteration, num_classes, weights_before):
"""
Interpolate between current weights and trained weights from this task
I.e. (weights_before - weights_after) is the meta-gradient
- iteration: current iteration - used for updating outer_stepsize
- num_classes: current classifier number of classes
- weights_before: state of weights before inner steps training
"""
weights_after = self.model.state_dict()
outer_stepsize = self.outer_stepsize * (
1 - iteration / self.args.n_iterations) # linear schedule
self.model.load_state_dict({
name: weights_before[name] +
(weights_after[name] - weights_before[name]) * outer_stepsize
for name in weights_before
})
def meta_training(self):
# Reptile training loop
total_loss = 0
try:
while self.current_iteration < self.args.n_iterations:
# Generate task
data, labels, original_labels, num_classes = self.task_generator.get_train_task(
args.num_classes)
weights_before = deepcopy(self.model.state_dict())
loss = self.inner_training(data, labels,
self.args.inner_iterations)
total_loss += loss
if self.current_iteration % self.args.log_every == 0:
print("-----------------------------")
print("iteration {}".format(
self.current_iteration + 1))
print("Loss: {:.3f}".format(total_loss /
(self.current_iteration + 1)))
print("Current task info: ")
print("\t- Number of classes: {}".format(num_classes))
print("\t- Batch size: {}".format(len(data)))
print("\t- Labels: {}".format(set(original_labels)))
self.test()
self._meta_gradient_update(self.current_iteration, num_classes,
weights_before)
self.current_iteration += 1
torch.save(self.model.state_dict(), self.args.model_path)
except KeyboardInterrupt:
print("Manual Interrupt...")
print("Saving to: {}".format(self.args.model_path))
torch.save(self.model.state_dict(), self.args.model_path)
joblib.dump(self.current_iteration,
"{}.iter".format(self.args.model_path),
compress=1)
def predict(self, x):
self.model.eval()
x = torch.tensor(x, dtype=torch.float, device=self.args.device)
outputs = self.model(x)
return outputs.cpu().data.numpy()
def test(self):
"""
Run tests
1. Create task from test set.
2. Reload model
3. Check accuracy on test set
4. Train for one or more iterations on one task
5. Check accuracy again on test set
"""
test_data, test_labels, _, _ = self.task_generator.get_test_task(
selected_labels=[1, 2, 3, 4,
5], num_samples=-1) # all available samples
predicted_labels = np.argmax(self.predict(test_data), axis=1)
accuracy = np.mean(1 * (predicted_labels == test_labels)) * 100
print(
"Accuracy before few shots learning (a.k.a. zero-shot learning): {:.2f}%\n----"
.format(accuracy))
weights_before = deepcopy(
self.model.state_dict()) # save snapshot before evaluation
for i in range(1, 5):
enroll_data, enroll_labels, _, _ = self.task_generator.get_enroll_task(
selected_labels=[1, 2, 3, 4, 5], num_samples=i)
self.inner_training(enroll_data, enroll_labels,
self.args.inner_iterations_test)
predicted_labels = np.argmax(self.predict(test_data), axis=1)
accuracy = np.mean(1 * (predicted_labels == test_labels)) * 100
print("Accuracy after {} shot{} learning: {:.2f}%)".format(
i, "" if i == 1 else "s", accuracy))
self.model.load_state_dict(weights_before) # restore from snapshot
batch_size=param['test_batch_size'],
shuffle=True)
model = LeNet()
model.load_state_dict(
torch.load('models/lenet_pretrained.pkl', map_location='cpu'))
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model.to(device)
print("--- Accuracy of Pretrained Model ---")
test(model, loader_test)
# pruning
masks = lenet_prune()
model.set_masks(masks)
print("--- Accuracy After Pruning ---")
test(model, loader_test)
# Retraining
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.RMSprop(model.parameters(),
lr=param['learning_rate'],
weight_decay=param['weight_decay'])
train(model, criterion, optimizer, param, loader_train)
print("--- Accuracy After Retraining ---")
test(model, loader_test)
prune_rate(model)
# Save and load the entire model
torch.save(model.state_dict(), 'models/lenet_pruned.pkl')
class Solver(object):
def __init__(self, config):
self.model = None
self.lr = config.lr
self.epochs = config.epoch
self.train_batch_size = config.trainBatchSize
self.test_batch_size = config.testBatchSize
self.criterion = None
self.optimizer = None
self.scheduler = None
self.device = None
self.cuda = config.cuda
self.train_loader = None
self.test_loader = None
def load_data(self):
train_transform = transforms.Compose(
[transforms.RandomHorizontalFlip(),
transforms.ToTensor()])
test_transform = transforms.Compose([transforms.ToTensor()])
train_set = torchvision.datasets.CIFAR10(root='./data',
train=True,
download=True,
transform=train_transform)
self.train_loader = torch.utils.data.DataLoader(
dataset=train_set, batch_size=self.train_batch_size, shuffle=True)
test_set = torchvision.datasets.CIFAR10(root='./data',
train=False,
download=True,
transform=test_transform)
self.test_loader = torch.utils.data.DataLoader(
dataset=test_set, batch_size=self.test_batch_size, shuffle=False)
def load_model(self):
if self.cuda:
self.device = torch.device('cuda')
cudnn.benchmark = True
else:
self.device = torch.device('cpu')
self.model = LeNet().to(self.device)
# self.model = AlexNet().to(self.device)
# self.model = VGG11().to(self.device)
# self.model = VGG13().to(self.device)
# self.model = VGG16().to(self.device)
# self.model = VGG19().to(self.device)
# self.model = GoogLeNet().to(self.device)
# self.model = resnet18().to(self.device)
# self.model = resnet34().to(self.device)
# self.model = resnet50().to(self.device)
# self.model = resnet101().to(self.device)
# self.model = resnet152().to(self.device)
# self.model = DenseNet121().to(self.device)
# self.model = DenseNet161().to(self.device)
# self.model = DenseNet169().to(self.device)
# self.model = DenseNet201().to(self.device)
# self.model = WideResNet(depth=28, num_classes=10).to(self.device)
self.optimizer = optim.Adam(self.model.parameters(), lr=self.lr)
self.scheduler = optim.lr_scheduler.MultiStepLR(self.optimizer,
milestones=[75, 150],
gamma=0.5)
self.criterion = nn.CrossEntropyLoss().to(self.device)
def train(self):
print("train:")
self.model.train()
train_loss = 0
train_correct = 0
total = 0
for batch_num, (data, target) in enumerate(self.train_loader):
data, target = data.to(self.device), target.to(self.device)
self.optimizer.zero_grad()
output = self.model(data)
feature = self.model.feature
# print('output.shape = {}, target.shape = {}, feature.shape = {}'.format(output.size(), target.size(), feature.size()))
loss = self.criterion(output, target)
loss.backward()
self.optimizer.step()
train_loss += loss.item()
prediction = torch.max(
output,
1) # second param "1" represents the dimension to be reduced
total += target.size(0)
# train_correct incremented by one if predicted right
train_correct += np.sum(
prediction[1].cpu().numpy() == target.cpu().numpy())
progress_bar(
batch_num, len(self.train_loader),
'Loss: %.4f | Acc: %.3f%% (%d/%d)' %
(train_loss / (batch_num + 1), 100. * train_correct / total,
train_correct, total))
return train_loss, train_correct / total
def test(self):
print("test:")
self.model.eval()
test_loss = 0
test_correct = 0
total = 0
with torch.no_grad():
for batch_num, (data, target) in enumerate(self.test_loader):
data, target = data.to(self.device), target.to(self.device)
output = self.model(data)
# CAM
# feature = self.model.feature
# print('feature: {}'.format(feature))
loss = self.criterion(output, target)
test_loss += loss.item()
prediction = torch.max(output, 1)
total += target.size(0)
test_correct += np.sum(
prediction[1].cpu().numpy() == target.cpu().numpy())
progress_bar(
batch_num, len(self.test_loader),
'Loss: %.4f | Acc: %.3f%% (%d/%d)' %
(test_loss / (batch_num + 1), 100. * test_correct / total,
test_correct, total))
return test_loss, test_correct / total
def save(self):
model_out_path = "model.pth"
torch.save(self.model, model_out_path)
print("Checkpoint saved to {}".format(model_out_path))
def run(self):
self.load_data()
print('Success loading data.')
self.load_model()
print('Success loading model.')
accuracy = 0
for epoch in range(1, self.epochs + 1):
self.scheduler.step(epoch)
print("\n===> epoch: %d/200" % epoch)
train_result = self.train()
print(train_result)
test_result = self.test()
accuracy = max(accuracy, test_result[1])
if epoch == self.epochs:
print("===> BEST ACC. PERFORMANCE: %.3f%%" % (accuracy * 100))
self.save()
from torchvision.transforms import ToTensor
from test import test_img
from torch.utils.tensorboard import SummaryWriter
if __name__ == '__main__':
args = args_parser()
args.device = torch.device('cuda:{}'.format(args.gpu))
batch_size = 256
trans_mnist = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])
train_dataset = MNIST('./data/mnist/', train=True, download=True, transform=trans_mnist)
test_dataset = MNIST('./data/mnist/', train=False, download=True, transform=trans_mnist)
train_loader = DataLoader(train_dataset, batch_size=batch_size)
test_loader = DataLoader(test_dataset, batch_size=batch_size)
model = LeNet().to(args.device)
sgd = SGD(model.parameters(), lr=1e-1)
cross_error = CrossEntropyLoss()
epoch = 100
writer = SummaryWriter('./runs/t_centerlize')
for _epoch in range(epoch):
epoch_loss = []
for idx, (train_x, train_label) in enumerate(train_loader):
train_x, train_label = train_x.to(args.device), train_label.to(args.device)
#label_np = np.zeros((train_label.shape[0], 10))
sgd.zero_grad()
predict_y = model(train_x.float())
_error = cross_error(predict_y, train_label.long())
_error.backward()
sgd.step()
epoch_loss.append(_error)
def main():
# Training settings
parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
parser.add_argument('--batch-size',
type=int,
default=64,
metavar='N',
help='input batch size for training (default: 64)')
parser.add_argument('--test-batch-size',
type=int,
default=1000,
metavar='N',
help='input batch size for testing (default: 1000)')
parser.add_argument('--epochs',
type=int,
default=10,
metavar='N',
help='number of epochs to train (default: 10)')
parser.add_argument('--lr',
type=float,
default=0.01,
metavar='LR',
help='learning rate (default: 0.01)')
parser.add_argument('--momentum',
type=float,
default=0.5,
metavar='M',
help='SGD momentum (default: 0.5)')
parser.add_argument('--no-cuda',
action='store_true',
default=False,
help='disables CUDA training')
parser.add_argument('--seed',
type=int,
default=1,
metavar='S',
help='random seed (default: 1)')
parser.add_argument(
'--log-interval',
type=int,
default=10,
metavar='N',
help='how many batches to wait before logging training status')
args = parser.parse_args()
use_cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
train_loader = torch.utils.data.DataLoader(datasets.MNIST(
'./data',
train=True,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])),
batch_size=args.batch_size,
shuffle=True,
**kwargs)
test_loader = torch.utils.data.DataLoader(datasets.MNIST(
'./data',
train=False,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])),
batch_size=args.test_batch_size,
shuffle=True,
**kwargs)
feature_blob = np.zeros([1, 16, 8, 8])
model = LeNet()
def hook(module, input, output):
global feature_blob
feature_blob = output
model._modules.get('conv2').register_forward_hook(hook)
model = torch.nn.DataParallel(model).cuda()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum)
for epoch in range(1, args.epochs + 1):
train(args, model, train_loader, optimizer, epoch)
cam(model, epoch)
test(args, model, test_loader)
generate_gif()
classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse',
'ship', 'truck')
# Model
print('==> Building model..')
model = LeNet(3, len(classes)).to(device)
if os.path.isfile(ckpt_file):
checkpoint = torch.load(ckpt_file)
model.load_state_dict(checkpoint['net'])
best_acc = checkpoint['acc']
start_epoch = checkpoint['epoch']
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=1e-3, momentum=0.9)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')
# Training
def train(epoch):
print('\nEpoch: %d' % epoch)
model.train()
train_loss = []
correct = 0
total = 0
for batch_idx, (inputs, targets) in enumerate(trainloader):
inputs, targets = inputs.to(device), targets.to(device)
optimizer.zero_grad()
outputs = model(inputs)
loss = criterion(outputs, targets)
