def main(config):
GPUManager.auto_chooce()
engine = DefaultClassificationEngine()
engine.hooks.update(
dict(
on_sample=on_sample_hook,
on_start=on_start_hook,
on_forward=on_forward_hook,
)
)
net = LeNet(n_channels=config.n_channels, size=config.size)
print(net)
if torch.cuda.is_available():
net = net.cuda()
train_loader = torch.utils.data.DataLoader(
config.dataset(train=True),
batch_size=config.batch_size, shuffle=True, num_workers=8
)
val_loader = torch.utils.data.DataLoader(
config.dataset(train=False),
batch_size=1000, shuffle=True, num_workers=8)
optimizer = optim.SGD(net.parameters(), lr=config.lr, momentum=0.9)
recorder = defaultdict(list)
recorder.update(
dict(
dataset_name=config.dataset.__name__.split("_")[0],
lr=config.lr,
batch_size=config.batch_size
)
)
pprint(recorder)
for epoch in range(config.maxepoch):
state=engine.train(network=net, iterator=train_loader, maxepoch=1, optimizer=optimizer)
recorder["train_loss"].append(state["loss_meter"].value())
recorder["train_acc"].append(state["acc_meter"].value())
state=engine.validate(network=net, iterator=val_loader)
recorder["val_loss"].append(state["loss_meter"].value())
recorder["val_acc"].append(state["acc_meter"].value())
filename = f"{recorder['dataset_name']}_" + time.strftime("%Y%m%d_%H%M%S", time.localtime())
with open(f"../result/{filename}.static", "wb") as f:
pickle.dump(recorder, f)
def main(args):
# Data Loader (Input Pipeline)
model_str = args.dataset + '_%s_' % args.model_type + args.noise_type + '_' + str(args.noise_rate) + '_' + str(args.seed)
txtfile = save_dir + "/" + model_str + ".txt"
nowTime = datetime.datetime.now().strftime('%Y-%m-%d-%H:%M:%S')
if os.path.exists(txtfile):
os.system('mv %s %s' % (txtfile, txtfile + ".bak-%s" % nowTime))
# Data Loader (Input Pipeline)
print('loading dataset...')
train_dataset, val_dataset, test_dataset = load_data(args)
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers,
drop_last=False,
shuffle=True)
val_loader = torch.utils.data.DataLoader(dataset=val_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers,
drop_last=False,
shuffle=False)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers,
drop_last=False,
shuffle=False)
# Define models
print('building model...')
if args.dataset == 'mnist':
clf1 = LeNet()
optimizer1 = torch.optim.SGD(clf1.parameters(), lr=learning_rate, weight_decay=args.weight_decay, momentum=0.9)
scheduler1 = MultiStepLR(optimizer1, milestones=[10, 20], gamma=0.1)
elif args.dataset == 'fmnist':
clf1 = resnet.ResNet50(input_channel=1, num_classes=10)
optimizer1 = torch.optim.SGD(clf1.parameters(), lr=learning_rate, weight_decay=args.weight_decay, momentum=0.9)
scheduler1 = MultiStepLR(optimizer1, milestones=[10, 20], gamma=0.1)
elif args.dataset == 'cifar10':
clf1 = resnet.ResNet50(input_channel=3, num_classes=10)
optimizer1 = torch.optim.SGD(clf1.parameters(), lr=learning_rate, weight_decay=args.weight_decay, momentum=0.9)
scheduler1 = MultiStepLR(optimizer1, milestones=[40, 80], gamma=0.1)
elif args.dataset == 'cifar100':
clf1 = resnet.ResNet50(input_channel=3, num_classes=100)
optimizer1 = torch.optim.SGD(clf1.parameters(), lr=learning_rate, weight_decay=args.weight_decay, momentum=0.9)
scheduler1 = MultiStepLR(optimizer1, milestones=[40, 80], gamma=0.1)
clf1.cuda()
with open(txtfile, "a") as myfile:
myfile.write('epoch train_acc1 val_acc1 test_acc1\n')
epoch = 0
train_acc1 = 0
# evaluate models with random weights
val_acc1 = evaluate(val_loader, clf1)
print('Epoch [%d/%d] Val Accuracy on the %s val data: Model1 %.4f %%' % (
epoch + 1, args.n_epoch, len(val_dataset), val_acc1))
test_acc1 = evaluate(test_loader, clf1)
print('Epoch [%d/%d] Test Accuracy on the %s test data: Model1 %.4f %%' % (
epoch + 1, args.n_epoch, len(test_dataset), test_acc1))
# save results
with open(txtfile, "a") as myfile:
myfile.write(str(int(epoch)) + ' ' + str(train_acc1) + ' ' + str(val_acc1) + ' ' + str(test_acc1) + "\n")
val_acc_list = []
test_acc_list = []
for epoch in range(0, args.n_epoch):
scheduler1.step()
print(optimizer1.state_dict()['param_groups'][0]['lr'])
clf1.train()
train_acc1 = train(train_loader, epoch, clf1, optimizer1, args)
val_acc1 = evaluate(val_loader, clf1)
val_acc_list.append(val_acc1)
test_acc1 = evaluate(test_loader, clf1)
test_acc_list.append(test_acc1)
# save results
print('Epoch [%d/%d] Test Accuracy on the %s test data: Model1 %.4f %% ' % (
epoch + 1, args.n_epoch, len(test_dataset), test_acc1))
with open(txtfile, "a") as myfile:
myfile.write(str(int(epoch)) + ' ' + str(train_acc1) + ' ' + str(val_acc1) + ' ' + str(test_acc1) + "\n")
id = np.argmax(np.array(val_acc_list))
test_acc_max = test_acc_list[id]
return test_acc_max
for p in model.parameters():
p.requires_grad_(False)
#change criterion for ImageNet-1k and CIFAR100 to Top5Criterion
criterion = lambda x, y: Top1Criterion(x, y, model)
if args.norm == 'L2':
norm = 2
elif args.norm == 'Linf':
norm = np.inf
elif args.norm == 'L0':
norm = 0
elif args.norm == 'L1':
norm = 1
if has_cuda:
model = model.cuda()
params = {
'bounds': (0, 1),
'dt': args.dt,
'alpha': args.alpha,
'beta': args.beta,
'gamma': args.gamma,
'max_outer': args.max_outer,
'max_inner': args.max_inner,
'T1': args.T1,
'T2': args.T2
}
attack = Attack(model, **params)
pred = output.data.max(1, keepdim=True)[1]
correct += pred.eq(target.data.view_as(pred)).cpu().sum()
test_loss /= len(validate_loader.dataset)
print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
test_loss, correct, len(validate_loader.dataset),
100. * correct / len(validate_loader.dataset)))
if args.lenet:
model = LeNet()
else:
model = Net()
if args.cuda:
model.cuda()
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)
model.train()
for epoch in range(args.epochs):
for batch_idx, (data, target) in enumerate(train_loader):
if args.cuda:
data, target = data.cuda(), target.cuda()
data, target = Variable(data), Variable(target)
optimizer.zero_grad()
output = model(data)
loss = F.nll_loss(output, target)
loss.backward()
optimizer.step()
if batch_idx % args.log_interval == 0:
transform=transforms.ToTensor(),
train=False)
# record
save_acc = open("record/MNIST_early_stopping_acc.txt", "w")
train_loss = []
train_acc = []
test_loss = []
test_acc = []
# cuda setting
train_args.cuda = not train_args.no_cuda and torch.cuda.is_available()
if train_args.cuda:
torch.cuda.manual_seed(
train_args.seed) # set a random seed for the current GPU
model.cuda() # move all model parameters to the GPU
optimizer = optim.SGD(model.parameters(),
lr=train_args.lr,
momentum=train_args.momentum)
# early stopping
check_correct = 0
check_count = 0
model_save_dir = 'record/'
def _iter_epoch(epoch, is_training):
if is_training:
model.train()
images = data_train.data.tolist()
labels = data_train.targets.tolist()
def train_lre(hyperparameters,
data_loader,
val_data,
val_labels,
test_loader,
n_val=5):
net, opt = build_model(hyperparameters)
plot_step = 10
accuracy_log = dict()
data_log = dict()
subselect_val = n_val * val_labels.unique().size()[0] != len(val_labels)
meta_net = LeNet(n_out=1)
for i in range(hyperparameters['num_iterations']):
t = time.time()
net.train()
# Line 2 get batch of data
image, labels, index = next(iter(data_loader))
# since validation data is small I just fixed them instead of building an iterator
# initialize a dummy network for the meta learning of the weights
meta_net.load_state_dict(net.state_dict())
if torch.cuda.is_available():
meta_net.cuda()
image = to_var(image, requires_grad=False)
labels = to_var(labels, requires_grad=False)
# Lines 4 - 5 initial forward pass to compute the initial weighted loss
y_f_hat = meta_net(image) # Line 4
cost = functional.binary_cross_entropy_with_logits(
y_f_hat, labels, reduce=False) # Line 5
eps = to_var(torch.zeros(cost.size())) # Line 5
l_f_meta = torch.sum(cost * eps)
meta_net.zero_grad()
# Line 6 perform a parameter update
grads = torch.autograd.grad(l_f_meta, (meta_net.params()),
create_graph=True)
meta_net.update_params(hyperparameters['lr'], source_params=grads)
# Line 8 - 10 2nd forward pass and getting the gradients with respect to epsilon
if subselect_val:
class_inds = list()
for c in val_labels.unique():
matching_inds = (val_labels == c).nonzero()
class_inds.append(matching_inds[np.random.permutation(
len(matching_inds))[:n_val]])
class_inds = torch.cat(class_inds)
val_input = val_data[class_inds].squeeze_(1)
val_output = val_labels[class_inds].squeeze_(1)
else:
val_input = val_data
val_output = val_labels
y_g_hat = meta_net(val_input)
l_g_meta = functional.binary_cross_entropy_with_logits(
y_g_hat, val_output)
grad_eps = torch.autograd.grad(l_g_meta, eps, only_inputs=True)[0]
# Line 11 computing and normalizing the weights
w_tilde = torch.clamp(-grad_eps, min=0)
norm_c = torch.sum(w_tilde)
if norm_c != 0:
w = w_tilde / norm_c
else:
w = w_tilde
# Lines 12 - 14 computing for the loss with the computed weights
# and then perform a parameter update
y_f_hat = net(image)
cost = functional.binary_cross_entropy_with_logits(y_f_hat,
labels,
reduce=False)
data_log[i] = pd.DataFrame(
data={
'w': w.numpy(),
'grad_eps': grad_eps.numpy(),
'pre_w_cost': cost.detach().numpy(),
'label': labels.numpy(),
'index': index.numpy(),
})
l_f = torch.sum(cost * w)
# print(data_log[i].head())
# print(data_log[i].tail())
opt.zero_grad()
l_f.backward()
opt.step()
# meta_l = smoothing_alpha * meta_l + (1 - smoothing_alpha) * l_g_meta.item()
# meta_losses_clean.append(meta_l / (1 - smoothing_alpha ** (i + 1)))
#
# net_l = smoothing_alpha * net_l + (1 - smoothing_alpha) * l_f.item()
# net_losses.append(net_l / (1 - smoothing_alpha ** (i + 1)))
#
print(time.time() - t)
if i % plot_step == 0:
print(i)
net.eval()
acc_df = {'preds': [], 'labels': [], 'index': []}
for itr, (test_img, test_label,
test_idx) in enumerate(test_loader):
test_img = to_var(test_img, requires_grad=False)
test_label = to_var(test_label, requires_grad=False)
output = net(test_img)
predicted = functional.sigmoid(output)
acc_df['preds'].extend(predicted.detach().numpy().tolist())
acc_df['labels'].extend(test_label.numpy().tolist())
acc_df['index'].extend(test_idx.numpy().tolist())
accuracy_log[i] = pd.DataFrame(acc_df).sort_values(
['labels', 'index']).set_index('index', drop=True)
data_log = pd.concat(data_log)
data_log.index.set_names('iteration', level=0, inplace=True)
accuracy_log = pd.concat(accuracy_log)
accuracy_log.index.set_names('iteration', level=0, inplace=True)
return data_log, accuracy_log
