def plot(label, num_filled, freq,
num_shells_range, interaction="normal"):
d = utils.load_all()
d = utils.filter_preferred_ml(d)
d = d[(d["method"] != "imsrg[f]+eom[n]") &
(d["method"] != "magnus_quads+eom") &
(d["interaction"] == interaction) &
(d["label"] == label) &
(d["num_filled"] == num_filled) &
(d["num_shells"] >= num_shells_range[0]) &
(d["num_shells"] <= num_shells_range[1]) &
(d["freq"] == 1.0)]
num_particles = num_filled * (num_filled + 1)
energy_type = {"ground": "ground state",
"add": "addition",
"rm": "removal"}[label]
fig, ax = plt.subplots()
fig.set_size_inches((4, 3))
for method, case in d.groupby("method"):
case = case.sort_values("num_shells")
xs = case["num_shells"].astype(int)
ys = case["energy"]
ax.plot(xs, ys, "-x", label=utils.METHOD_LABEL[method],
color=utils.METHOD_COLOR[method])
ax.get_xaxis().set_major_locator(
matplotlib.ticker.MaxNLocator(integer=True))
ax.set_xlabel("K (number of shells)")
ax.set_ylabel("E (energy)")
ax.legend()
fig.tight_layout()
utils.savefig(fig,
"by-num-shells-{num_particles}-{num_filled}-"
"{label}-{ml}-{interaction}"
.format(**locals()))
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data loading code
traindir = os.path.join(args.data, 'train')
valdir = os.path.join(args.data, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_loader = torch.utils.data.DataLoader(
datasets.ImageFolder(traindir, transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
])),
batch_size=args.train_batch, shuffle=True,
num_workers=args.workers, pin_memory=True, worker_init_fn=random.seed)
val_loader = torch.utils.data.DataLoader(
datasets.ImageFolder(valdir, transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size=args.test_batch, shuffle=False,
num_workers=args.workers, pin_memory=True)
# create model
if args.pretrained:
print("=> using pre-trained model '{}'".format(args.arch))
model = models.__dict__[args.arch](pretrained=True)
elif args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
baseWidth=args.base_width,
cardinality=args.cardinality,
)
else:
print("=> creating model '{}'".format(args.arch))
model = models.__dict__[args.arch]()
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters())/1000000.0))
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay)
# Resume
title = 'ImageNet-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!'
# args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.TOP1', 'Valid Acc.TOP5'])
lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,
milestones=args.schedule, gamma=args.gamma, last_epoch=start_epoch - 1)
if args.evaluate:
# print('\nEvaluation only')
test_loss, test_acc, test_acc_top5 = test(val_loader, model, criterion, start_epoch, use_cuda)
print(' Test Loss: %.8f, Test Acc TOP1: %.2f, Test Acc TOP5: %.2f' % (test_loss, test_acc, test_acc_top5))
# return
# Train and val
for epoch in range(start_epoch, args.epochs):
lr_scheduler.step()
state['lr'] = optimizer.param_groups[0]['lr']
add_summary_value(tb_summary_writer, 'learning_rate', state['lr'], epoch + 1)
print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch+1, use_cuda)
test_loss, test_acc, test_acc_top5 = test(val_loader, model, criterion, epoch+1, use_cuda)
# append logger file
logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc, test_acc_top5])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'acc_top5': test_acc_top5,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
}, is_best, checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
def main():
global best_acc
mp.set_start_method('spawn')
e = mp.Event()
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data
print('==> Preparing dataset %s' % args.dataset)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
if args.dataset == 'cifar10':
dataloader = datasets.CIFAR10
num_classes = 10
else:
dataloader = datasets.CIFAR100
num_classes = 100
trainset = dataloader(root='./data',
train=True,
download=True,
transform=transform_train)
trainset_small, trainset_large = trainset, trainset
# torch.utils.data.random_split(trainset, [len(trainset)//2, len(trainset) - (len(trainset)//2)])
print("len trainset_large: ", len(trainset_large))
print("len trainset_small: ", len(trainset_small))
trainloader_gpu0 = data.DataLoader(trainset_small,
batch_size=args.train_batch,
shuffle=True,
num_workers=args.workers)
trainloader_gpu1 = data.DataLoader(trainset_large,
batch_size=args.train_batch,
shuffle=True,
num_workers=args.workers)
testset = dataloader(root='./data',
train=False,
download=False,
transform=transform_test)
testloader = data.DataLoader(testset,
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers)
# Model
print("==> creating model '{}'".format(args.arch))
if args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
cardinality=args.cardinality,
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.startswith('densenet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
growthRate=args.growthRate,
compressionRate=args.compressionRate,
dropRate=args.drop,
)
elif args.arch.startswith('wrn'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.endswith('resnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
block_name=args.block_name,
)
else:
model = models.__dict__[args.arch](num_classes=num_classes)
print(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
criterion = nn.CrossEntropyLoss()
# Resume
title = 'cifar-10-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch,
use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# clone models
save_model(model, 0)
save_model(model, 1)
model0 = load_model(0)
model1 = load_model(1)
queue01 = mp.Queue()
queue10 = mp.Queue()
# training
model0.share_memory()
model1.share_memory()
train_gpu1 = mp.spawn(train_copy,
args=(trainloader_gpu1, model1, criterion, use_cuda,
gpu1, e, args, testloader, queue01, queue10),
join=False)
train_gpu0 = mp.spawn(train,
args=(trainloader_gpu0, model0, criterion, use_cuda,
gpu0, e, args, testloader, queue10, queue01),
join=False)
train_gpu1.join()
train_gpu0.join()
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
if question == '2':
dataset = load_dataset('animals.pkl')
X = dataset['X'].astype(float)
animals = dataset['animals']
n, d = X.shape
f1, f2 = np.random.choice(d, size=2, replace=False)
plt.figure()
plt.scatter(X[:, f1], X[:, f2])
plt.xlabel("$x_{%d}$" % f1)
plt.ylabel("$x_{%d}$" % f2)
for i in range(n):
plt.annotate(animals[i], (X[i, f1], X[i, f2]))
utils.savefig('two_random_features.png')
elif question == '2.2':
dataset = load_dataset('animals.pkl')
X = dataset['X'].astype(float)
animals = dataset['animals']
n, d = X.shape
# standardize columns
X = utils.standardize_cols(X)
k = 20
model = PCA(k)
model.fit(X)
Z = model.compress(X)
f1, f2 = np.random.choice(k, size=2, replace=False)
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data
print('==> Preparing dataset %s' % args.dataset)
if args.img_size == 32:
if args.DA:
print('use DA')
transform_train = transforms.Compose([
transforms.RandomHorizontalFlip(), # with p = 0.5
transforms.ToTensor(
), # it must be this guy that makes it CHW again
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
else:
print('no DA')
transform_train = transforms.Compose([
transforms.ToTensor(
), # it must be this guy that makes it CHW again
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
else:
if args.DA:
print('use DA')
transform_train = transforms.Compose([
transforms.RandomHorizontalFlip(), # with p = 0.5
transforms.Resize(args.img_size),
transforms.ToTensor(
), # it must be this guy that makes it CHW again
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
else:
print('no DA')
transform_train = transforms.Compose([
transforms.Resize(args.img_size),
transforms.ToTensor(
), # it must be this guy that makes it CHW again
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.Resize(args.img_size),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
if args.dataset == 'cifar10':
dataloader = CIFAR10
num_classes = 10
else:
dataloader = CIFAR100
num_classes = 100
trainset = dataloader(root='/BS/yfan/work/data/cifar100_fp',
train=True,
download=False,
transform=transform_train)
trainloader = data.DataLoader(trainset,
batch_size=args.train_batch,
shuffle=True,
num_workers=args.workers)
testset = dataloader(root='/BS/yfan/work/data/cifar100_fp',
train=False,
download=False,
transform=transform_test)
testloader = data.DataLoader(testset,
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers)
# Model
print("==> creating model '{}'".format(args.arch))
if args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
cardinality=args.cardinality,
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.startswith('wrn'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.endswith('resnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
)
elif args.arch.startswith('resnet50'):
model = models.__dict__[args.arch](num_classes=num_classes,
include_top=True,
dropout_rate=0,
layer=args.layer)
elif args.arch.startswith('d1_resnet50'):
model = models.__dict__[args.arch](
num_classes=num_classes,
include_top=True,
dropout_rate=0,
layer=args.layer,
is_shuff=False # TODO: check
)
elif args.arch.endswith('vgg16'):
model = models.__dict__[args.arch](num_classes=num_classes,
include_top=True,
dropout_rate=0,
layer=args.layer)
elif args.arch.endswith('vgg16_sa'):
model = models.__dict__[args.arch](num_classes=num_classes,
include_top=True,
dropout_rate=0,
layer=args.layer)
elif args.arch.endswith('vgg16_1d'):
model = models.__dict__[args.arch](num_classes=num_classes,
include_top=True,
dropout_rate=0,
layer=args.layer,
is_shuff=False)
elif args.arch.endswith("densenet_1d"):
model = models.__dict__[args.arch](num_classes=num_classes,
drop_rate=0,
layer=args.layer)
else:
model = models.__dict__[args.arch](num_classes=num_classes)
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = False # TODO: for deterministc result, this has to be false
print(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
# for name, param in model.named_parameters():
# print(name)
# for name in model.named_modules():
# print(name)
# for param in model.parameters():
# print(param)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# optimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=args.weight_decay)
# Resume
title = 'cifar-10-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch,
use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(trainloader, model, criterion, optimizer,
epoch, use_cuda)
test_loss, test_acc = test(testloader, model, criterion, epoch,
use_cuda)
# append logger file
logger.append(
[state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.save):
mkdir_p(args.save)
# Data
print('==> Preparing dataset %s' % args.dataset)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
if args.dataset == 'cifar10':
dataloader = datasets.CIFAR10
elif args.dataset == 'cifar100':
dataloader = datasets.CIFAR100
else:
raise ValueError(
'Expect dataset to be either CIFAR-10 or CIFAR-100 but got {}'.
format(args.dataset))
trainset = dataloader(root='./data',
train=True,
download=True,
transform=transform_train)
trainloader = data.DataLoader(trainset,
batch_size=args.train_batch,
shuffle=True,
num_workers=args.workers)
testset = dataloader(root='./data',
train=False,
download=False,
transform=transform_test)
testloader = data.DataLoader(testset,
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers)
# Model
print("==> creating model '{}'".format(args.arch))
model = arch_module.__dict__[args.arch](dataset=args.dataset)
cudnn.benchmark = True
print(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
lr_scheduler = optim.lr_scheduler.MultiStepLR(optimizer,
milestones=args.schedule,
gamma=args.gamma)
# Resume
title = 'cifar-10-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.save = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_prec1']
start_epoch = checkpoint['epoch']
model = arch_module.__dict__[args.arch](dataset=args.dataset,
cfg=checkpoint['cfg'])
# load the state dict of saved checkpoint
# turn the flag off to train from scratch
if args.load_model:
print('===> Resuming the state dict of saved model')
model.load_state_dict(checkpoint['state_dict'])
else:
print('===> Skip loading state dict of saved model')
# finetune a pruned network
if args.load_optimizer and ('optimizer' in checkpoint.keys()):
print('===> Resuming the state dict of saved checkpoint')
optimizer.load_state_dict(checkpoint['optimizer'])
else:
print('===> Skip loading the state dict of saved optimizer')
# if the log file is already exist then append the log to it
if os.path.isfile('log.txt'):
logger = Logger(os.path.join(args.save, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.save, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
else:
# training from scratch
logger = Logger(os.path.join(args.save, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
if use_cuda:
model = model.cuda()
# evaluate the results on test set
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch,
use_cuda)
inp = torch.rand(1, 3, 32, 32)
if use_cuda:
inp = inp.cuda()
flops, params = get_model_complexity_info(model, (3, 32, 32),
as_strings=True,
print_per_layer_stat=True)
print('{:<30} {:<8}'.format('Computational complexity: ', flops))
print('{:<30} {:<8}'.format('Number of parameters: ', params))
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
current_lr = next(iter(optimizer.param_groups))['lr']
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, current_lr))
train_loss, train_acc = train(trainloader, model, criterion, optimizer,
lr_scheduler, epoch, use_cuda)
test_loss, test_acc = test(testloader, model, criterion, epoch,
use_cuda)
# append logger file
logger.append([current_lr, train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_prec1': test_acc,
'optimizer': optimizer.state_dict(),
'cfg': model.cfg
},
is_best,
checkpoint=args.save)
logger.close()
logger.plot()
savefig(os.path.join(args.save, 'log.eps'))
print('Best acc:')
print(best_acc)
X = dataset['X'].astype(float)
animals = dataset['animals']
n, d = X.shape
print("n =", n)
print("d =", d)
f1, f2 = np.random.choice(d, size=2, replace=False)
plt.figure()
plt.scatter(X[:, f1], X[:, f2])
plt.xlabel("$x_{%d}$" % f1)
plt.ylabel("$x_{%d}$" % f2)
for i in range(n):
plt.annotate(animals[i], (X[i, f1], X[i, f2]))
utils.savefig('two_random_features.png')
elif question == '1.3':
dataset = load_dataset('animals.pkl')
X = dataset['X'].astype(float)
animals = dataset['animals']
n, d = X.shape
model = MDS(n_components=2)
Z = model.compress(X)
fig, ax = plt.subplots()
ax.scatter(Z[:, 0], Z[:, 1])
plt.ylabel('z2')
plt.xlabel('z1')
def plot_fits(plot,
data,
get_fit_range,
badness_threshold,
x_col,
x_label,
y_col,
y_label,
absdydx_label,
title_cols,
get_title,
get_fn,
color_cols,
get_color,
get_color_label,
get_dmc,
dmc_label,
dmc_yerr_col=None):
# continuous x range for plotting continuous functions
x_c = np.linspace(data[x_col].min() - 1, data[x_col].max() + 1, 250)
[(title_key, gg)] = utils.groupby(data, title_cols)
fig, ax = plt.subplots(2)
fig.set_size_inches(8, 10) # otherwise the text will get obscured
y_range = np.array([np.nan, np.nan])
fit_results = {}
for color_key, g in utils.groupby(gg, color_cols):
logging.info(f"plot_fits: method: {color_key['method']}")
color = get_color(color_key["method"])
label = get_color_label(color_key["method"])
g = g.sort_values([x_col])
d = g.rename(columns={x_col: "x", y_col: "y"})
deriv_d = differentiate(d, "x", "y", "dydx")
x = deriv_d["x"]
dydx = deriv_d["dydx"]
ax[0].plot(x, abs(dydx), "x", label=label, color=color)
ax[1].plot(d["x"], d["y"], "x", label=label, color=color)
utils.update_range(y_range, d["y"])
fit_range = get_fit_range(color_key["method"])
fit_range = (max(fit_range[0], d["x"].min()),
min(fit_range[1], d["x"].max()))
if fit_range[1] < fit_range[0]:
continue
fit_range = fit_range + np.array([-0.2, 0.2]) # to make it look nicer
ax[0].axvspan(fit_range[0], fit_range[1], alpha=0.05, color=color)
ax[1].axvspan(fit_range[0], fit_range[1], alpha=0.05, color=color)
d_subset = d[d["x"].between(*fit_range)]
deriv_subset = deriv_d[deriv_d["x"].between(*fit_range)]
if len(deriv_subset) < 2:
continue
fit = do_fit(d_subset, deriv_subset,
badness_threshold=badness_threshold)
if fit is None:
continue
fit_result = {
"num_points": len(d["x"])
}
fit_result.update(fit)
fit_results[color_key["method"]] = fit_result
outliers = deriv_subset.loc[fit["logderiv"]["outliers"]]
ax[0].plot(outliers["x"], abs(outliers["dydx"]), "o",
markerfacecolor="none",
label="", color="red")
for stage, result in fit.items():
if stage == "fixedab":
continue # fixedab yields the same plot here as logderiv
a = result["coefficient"]
b = result["exponent"]
b_err = result.get("exponent_err", None)
dydx_c = a * b * x_c ** (b - 1.0)
ax[0].plot(
x_c, abs(dydx_c), linestyle=STAGE_TO_LINESTYLE[stage],
label=label + " " + fit_label(stage, b, b_err),
color=color)
for stage, result in fit.items():
if "constant" not in result:
continue
a = result["coefficient"]
b = result["exponent"]
c = result["constant"]
b_err = result.get("exponent_err", None)
y_c = a * x_c ** b + c
ax[1].plot(
x_c, y_c, linestyle=STAGE_TO_LINESTYLE[stage],
label=label + " " + fit_label(stage, b, b_err),
color=color)
if b < 0:
ax[1].axhline(c, linestyle=":", color=color)
else:
logging.warn(f"plot_fits: {stage}.b >= 0: no asymptotic result")
utils.update_range(y_range, c)
g = get_dmc(**title_key)
if len(g):
y = g[y_col].iloc[0]
if dmc_yerr_col is not None:
y_err = g[dmc_yerr_col].iloc[0]
ax[1].axhspan(y - y_err, y + y_err, alpha=0.4,
color="black", label=dmc_label)
utils.update_range(y_range, [y - y_err, y + y_err])
# add an extra line to make sure it's visible
ax[1].axhline(y, alpha=0.4, color="black")
utils.update_range(y_range, [y])
ax[0].set_xlabel(x_label)
ax[0].set_ylabel(absdydx_label)
ax[0].set_xscale("log")
ax[0].set_yscale("log")
ax[0].set_title(get_title(**title_key))
box = ax[0].get_position()
ax[0].set_position([box.x0, box.y0, box.width * 0.6, box.height])
ax[0].legend(bbox_to_anchor=(1, 1.0))
ax[1].legend()
ax[1].set_xlabel(x_label)
ax[1].set_ylabel(y_label)
ax[1].set_ylim(*utils.expand_range(y_range, 0.05))
box = ax[1].get_position()
ax[1].set_position([box.x0, box.y0, box.width * 0.6, box.height])
ax[1].legend(bbox_to_anchor=(1, 1.0))
ax[1].get_xaxis().set_major_locator(
matplotlib.ticker.MaxNLocator(integer=True))
if plot:
fn = get_fn(**title_key)
settings_fn = os.path.join("plot_settings", fn + ".json")
settings = utils.load_json(settings_fn) or {"ax1": {}, "ax2": {}}
fit_results_fn = os.path.join("fit_results", fn + ".json")
def save_settings():
utils.save_json(settings_fn, settings)
utils.sync_axes_lims(ax[0], settings["ax1"], save_settings)
utils.sync_axes_lims(ax[1], settings["ax2"], save_settings)
utils.savefig(fig, fn)
return fit_results
def main():
global best_loss
start_epoch = 0 # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
model = models.CycleTime(class_num=params['classNum'],
trans_param_num=3,
pretrained=args.pretrained_imagenet,
temporal_out=params['videoLen'],
T=args.T,
hist=args.hist)
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = False
print(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
criterion = nn.CrossEntropyLoss().cuda()
if args.optim == 'adam':
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
betas=(args.momentum, 0.999),
weight_decay=args.weight_decay)
else:
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay,
momentum=0.95)
print('weight_decay: ' + str(args.weight_decay))
print('beta1: ' + str(args.momentum))
if len(args.pretrained) > 0:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.pretrained), 'Error: no checkpoint directory found!'
checkpoint = torch.load(args.pretrained)
partial_load(checkpoint['state_dict'], model)
# model.load_state_dict(checkpoint['state_dict'], strict=False)
del checkpoint
title = 'videonet'
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
start_epoch = checkpoint['epoch']
partial_load(checkpoint['state_dict'], model)
logger = Logger(os.path.join(args.checkpoint, 'log-resume.txt'),
title=title)
logger.set_names(
['Learning Rate', 'Train Loss', 'Theta Loss', 'Theta Skip Loss'])
del checkpoint
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names(
['Learning Rate', 'Train Loss', 'Theta Loss', 'Theta Skip Loss'])
train_loader = torch.utils.data.DataLoader(vlog.VlogSet(
params, is_train=True, frame_gap=args.frame_gap),
batch_size=params['batchSize'],
shuffle=True,
num_workers=args.workers,
pin_memory=True)
# Train and val
for epoch in range(start_epoch, args.epochs):
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
train_loss, theta_loss, theta_skip_loss = train(
train_loader, model, criterion, optimizer, epoch, use_cuda, args)
# append logger file
logger.append([state['lr'], train_loss, theta_loss, theta_skip_loss])
if epoch % 1 == 0:
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
},
checkpoint=args.checkpoint)
logger.close()
savefig(os.path.join(args.checkpoint, 'log.eps'))
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data
print('==> Preparing dataset %s' % args.dataset)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
if args.dataset == 'cifar10':
dataloader = datasets.CIFAR10
num_classes = 10
else:
dataloader = datasets.CIFAR100
num_classes = 100
trainset = dataloader(root='/home/amir/cifar-10',
train=True,
download=True,
transform=transform_train)
trainloader = data.DataLoader(trainset,
batch_size=args.train_batch,
shuffle=True,
num_workers=args.workers)
testset = dataloader(root='/home/amir/cifar-10',
train=False,
download=False,
transform=transform_test)
testloader = data.DataLoader(testset,
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers)
# Model
print("==> creating model '{}'".format(args.arch))
if args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
cardinality=args.cardinality,
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.startswith('densenet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
growthRate=args.growthRate,
compressionRate=args.compressionRate,
dropRate=args.drop,
)
elif args.arch.startswith('wrn'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.endswith('resnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
)
else:
model = models.__dict__[args.arch](num_classes=num_classes)
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
print(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
criterion = nn.CrossEntropyLoss()
params_to_optimize = [p for p in model.parameters() if p.requires_grad]
p1 = []
p2 = []
if type(model) is nn.DataParallel:
m = model.module
else:
m = model
for k, v in m.named_children():
if 'pooling_convolution' not in k:
p1.append(v.parameters())
else:
p2.append(v.parameters())
p1 = itertools.chain(*p1)
p2 = itertools.chain(*p2)
optimizer = optim.SGD([{
'params': p1
}, {
'params': p2,
'lr': 1e-3
}],
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# Resume
title = 'cifar-10-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch,
use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(trainloader, model, criterion, optimizer,
epoch, use_cuda)
test_loss, test_acc = test(testloader, model, criterion, epoch,
use_cuda)
# append logger file
logger.append(
[state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
train_loss, train_dice = run_epoch(
train_loader,
net,
phase="train",
optimizer=optimizer,
criterion=criterion,
epoch=epoch,
args=args,
)
val_loss, val_dice = run_epoch(valid_loader,
net,
phase="valid",
criterion=criterion,
epoch=epoch,
args=args)
# scheduler.step(val_loss)
lr = optimizer.param_groups[0]["lr"]
logger.append([lr, train_loss, val_loss, train_dice, val_dice])
save_ckp(net, optimizer, epoch + 1, args)
if optimizer.param_groups[0]["lr"] <= (args.lr / 100):
break
logger.close()
logger.plot(logger.names[1:3])
savefig(os.path.join(args.ckp, "0-loss-log.eps"))
logger.plot(logger.names[-2:])
savefig(os.path.join(args.ckp, "0-dice-log.eps"))
def main():
#####################
# START SETUP LOGGING
foldername = str(uuid.uuid4())
isplus = '+' if not args.no_augment else ''
savefolder = 'results/%s/%s%s/%s' % (args.arch, args.dataset, isplus, foldername)
os.system('mkdir -p %s' % savefolder)
args.checkpoint = savefolder
time.sleep(5) # wait for directory creation
print 'folder is ', foldername
# use sys.stdout to log to file
orig_stdout = sys.stdout
logfilename = '%s/log.txt' % (savefolder)
logfile = file(logfilename, 'w')
if not args.no_print:
print 'Printing to file %s' % logfilename
sys.stdout = logfile
else:
print 'Printing to stdout'
backupfname = '%s/code_snapshot_%s.zip' % (savefolder, str(datetime.now()))
backupfname = backupfname.replace(' ','_')
backup_code(backupfname, '.', ['.py'], ['result', 'log',])
print args
# END SETUP LOGGING
###################
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data
print('==> Preparing dataset %s' % args.dataset)
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
if args.no_augment:
print 'NO DATA AUGMENTATION'
transform_train = transform_test
else:
print 'USE DATA AUGMENTATION'
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
if args.dataset == 'cifar10':
dataloader = datasets.CIFAR10
num_classes = 10
else:
dataloader = datasets.CIFAR100
num_classes = 100
trainset = dataloader(root='./data', train=True, download=True, transform=transform_train)
testset = dataloader(root='./data', train=False, download=False, transform=transform_test)
if args.validation:
# select random subset for validation
N = len(trainset)
train_size = int(N*.9) # use 90 % of training set for training
valid_size = N-train_size
print 'number of training examples is %i/%i' % (train_size,N)
indices = torch.randperm(N)
train_indices = indices[:train_size]
valid_indices = indices[train_size:]
assert(set(train_indices).isdisjoint(set(valid_indices)))
trainloader = data.DataLoader(trainset, batch_size=args.train_batch, sampler=SubsetRandomSampler(train_indices), num_workers=args.workers)
testloader = data.DataLoader(trainset, batch_size=args.test_batch, sampler=SubsetRandomSampler(valid_indices), num_workers=args.workers)
else:
trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers)
testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers)
# Model
print("==> creating model '{}'".format(args.arch))
if args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
cardinality=args.cardinality,
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.startswith('densenet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
growthRate=args.growthRate,
compressionRate=args.compressionRate,
dropRate=args.drop,
)
elif args.arch.startswith('wrn'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.endswith('resnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
)
else:
model = models.__dict__[args.arch](num_classes=num_classes)
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters())/1000000.0))
# use OLE loss?
print 'lambda_ =', args.lambda_
lambda_ = args.lambda_
global use_OLE
print args
if lambda_>0:
use_OLE = True
else:
use_OLE = False
if use_OLE:
criterion = [nn.CrossEntropyLoss()] + [OLELoss(lambda_=args.lambda_)]
else:
criterion = [nn.CrossEntropyLoss()]
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay)
# Resume
title = 'cifar-10-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.join(os.path.dirname(args.resume), 'fine_tune')
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = 0 # checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
# logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True)
# logger = Logger(os.path.join(args.checkpoint, 'log_finetune.txt'), title=title)
# logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.'])
else:
pass
# logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
# logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.'])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f, lambda_: %f' % (epoch + 1, args.epochs, state['lr'], args.lambda_))
train_loss, train_acc = train(trainloader, model, criterion, optimizer, epoch, use_cuda)
test_loss, test_acc = test(testloader, model, criterion, epoch, use_cuda)
# append logger file
# logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer' : optimizer.state_dict(),
}, is_best, checkpoint=args.checkpoint)
# logger.close()
# logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
#############
# END LOGGING
sys.stdout = orig_stdout
logfile.close()
print '---'
print 'saved results to ', savefolder
print('Done!')
def main():
global args, best_prec1
args = parser.parse_args()
conf_name = args.data
conf_name += '_ortho' if args.ortho else ''
conf_name += ('_pre_' +
args.net_type) if args.net_type != 'default' else ''
args.checkpoint = os.path.join(args.checkpoint, conf_name)
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
model = models.Net(extractor_type=args.net_type).cuda()
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
extractor_params = list(map(id, model.extractor.parameters()))
classifier_params = filter(lambda p: id(p) not in extractor_params,
model.parameters())
optimizer = torch.optim.SGD([{
'params': model.extractor.parameters()
}, {
'params': classifier_params,
'lr': args.lr * 10
}],
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=30,
gamma=0.1)
# optionally resume from a checkpoint
title = 'CUB'
if args.resume:
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
cudnn.benchmark = True
print(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
# Data loading code
normalize = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
train_dataset = loader.ImageLoader(args.data,
transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]),
train=True)
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
val_loader = torch.utils.data.DataLoader(loader.ImageLoader(
args.data,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
if args.evaluate:
validate(val_loader, model, criterion)
return
for epoch in range(args.start_epoch, args.epochs):
scheduler.step()
lr = optimizer.param_groups[1]['lr']
print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, lr))
# train for one epoch
train_loss, train_acc = train(train_loader, model, criterion,
optimizer, epoch, args)
# evaluate on validation set
test_loss, test_acc = validate(val_loader, model, criterion)
# append logger file
logger.append([lr, train_loss, test_loss, train_acc, test_acc])
# remember best prec@1 and save checkpoint
is_best = test_acc > best_prec1
best_prec1 = max(test_acc, best_prec1)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_prec1)
def main():
global args, best_prec1
args = parser.parse_args()
if args.seed is not None:
random.seed(args.seed)
torch.manual_seed(args.seed)
cudnn.deterministic = True
warnings.warn('You have chosen to seed training. '
'This will turn on the CUDNN deterministic setting, '
'which can slow down your training considerably! '
'You may see unexpected behavior when restarting '
'from checkpoints.')
args.distributed = args.world_size > 1
if args.distributed:
dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
world_size=args.world_size)
# create model
print("=> creating model '{}'".format(args.arch))
model = models.__dict__[args.arch](width_mult=args.width_mult)
if not args.distributed:
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model, device_ids=[0]).cuda()
else:
model.cuda()
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[0])
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(), args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# optionally resume from a checkpoint
title = 'ImageNet-' + args.arch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})"
.format(args.resume, checkpoint['epoch']))
args.checkpoint = os.path.dirname(args.resume)
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True)
else:
print("=> no checkpoint found at '{}'".format(args.resume))
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.'])
cudnn.benchmark = True
# Data loading code
if args.data_backend == 'pytorch':
get_train_loader = get_pytorch_train_loader
get_val_loader = get_pytorch_val_loader
elif args.data_backend == 'dali-gpu':
get_train_loader = get_dali_train_loader(dali_cpu=False)
get_val_loader = get_dali_val_loader()
elif args.data_backend == 'dali-cpu':
get_train_loader = get_dali_train_loader(dali_cpu=True)
get_val_loader = get_dali_val_loader()
else:
raise ValueError
train_loader, train_loader_len = get_train_loader(args.data, args.batch_size, workers=args.workers, input_size=args.input_size)
val_loader, val_loader_len = get_val_loader(args.data, args.batch_size, workers=args.workers, input_size=args.input_size)
if args.evaluate:
from collections import OrderedDict
if os.path.isfile(args.weight):
print("=> loading pretrained weight '{}'".format(args.weight))
source_state = torch.load(args.weight)
target_state = OrderedDict()
for k, v in source_state.items():
if k[:7] != 'module.':
k = 'module.' + k
target_state[k] = v
model.load_state_dict(target_state)
else:
print("=> no weight found at '{}'".format(args.weight))
if args.prune:
from xavier_lib import StatisticManager
import time
manager = StatisticManager()
manager(model)
# train_loss, train_acc = train(train_loader, train_loader_len, model, criterion, optimizer, 150)
validate(val_loader, val_loader_len, model, criterion)
for _ in range(1):
manager.computer_score()
manager.prune_local(200)
manager.pruning_overview()
validate(val_loader, val_loader_len, model, criterion)
for i in range(30):
train(train_loader, train_loader_len, model, criterion, optimizer, i*5)
manager.reset()
validate(val_loader, val_loader_len, model, criterion)
log_time = time.strftime("%Y-%m-%d_%H-%M-%S")
torch.save(model.state_dict(), './ckp_for_pruning/'+log_time+'.pth')
else:
validate(val_loader, val_loader_len, model, criterion)
return
for epoch in range(args.start_epoch, args.epochs):
if args.distributed:
train_sampler.set_epoch(epoch)
print('\nEpoch: [%d | %d]' % (epoch + 1, args.epochs))
# train for one epoch
train_loss, train_acc = train(train_loader, train_loader_len, model, criterion, optimizer, epoch)
# evaluate on validation set
val_loss, prec1 = validate(val_loader, val_loader_len, model, criterion)
lr = optimizer.param_groups[0]['lr']
# append logger file
logger.append([lr, train_loss, val_loss, train_acc, prec1])
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
save_checkpoint({
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer' : optimizer.state_dict(),
}, is_best, checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best accuracy:')
print(best_prec1)
def main():
args = parser.parse_args()
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
if args.remark != None:
args.remark = args.remark
else:
args.remark = args.dataset + "-" + args.task + "-" + args.norm
if args.dataset == "shapenet":
args.num_class = 16
else:
args.num_class = 40
def log_string(str):
logger.info(str)
print(str)
'''HYPER PARAMETER'''
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
'''CREATE DIR'''
timestr = str(datetime.datetime.now().strftime('%Y-%m-%d_%H-%M'))
experiment_dir = Path('/data-x/g12/zhangjie/3dIP/exp/v1')
experiment_dir.mkdir(exist_ok=True)
experiment_dir = experiment_dir.joinpath('pruning')
experiment_dir.mkdir(exist_ok=True)
experiment_dir = experiment_dir.joinpath(args.remark + "_" + timestr)
experiment_dir.mkdir(exist_ok=True)
checkpoints_dir = experiment_dir.joinpath('checkpoints/')
checkpoints_dir.mkdir(exist_ok=True)
log_dir = experiment_dir.joinpath('logs/')
log_dir.mkdir(exist_ok=True)
'''LOG_curve'''
title = args.dataset + "-" + args.task + "-" + args.norm + "-" + "Pruning"
logger_loss = Logger(os.path.join(log_dir, 'log_loss.txt'), title=title)
logger_loss.set_names(['Valid Loss', ' Trigger Valid Loss'])
logger_acc = Logger(os.path.join(log_dir, 'log_acc.txt'), title=title)
logger_acc.set_names(['Valid Acc.', 'Trigger Valid Acc.'])
'''LOG''' #创建log文件
logger = logging.getLogger("Model") #log的名字
logger.setLevel(logging.INFO)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
file_handler = logging.FileHandler('%s/%s.txt' % (log_dir, args.model))
file_handler.setLevel(logging.INFO) #log的最低等级
file_handler.setFormatter(formatter)
logger.addHandler(file_handler) #log文件名
log_string('PARAMETER ...')
log_string(args)
'''DATA LOADING'''
log_string('Load pruning test dataset ...')
if args.dataset == "shapenet":
testDataLoader = getData.get_dataLoader(train=False,
Shapenet=True,
batchsize=args.batch_size)
triggerDataLoader = getData2.get_dataLoader(Shapenet=True,
batchsize=args.batch_size)
else:
testDataLoader = getData.get_dataLoader(train=False,
Shapenet=False,
batchsize=args.batch_size)
triggerDataLoader = getData2.get_dataLoader(Shapenet=False,
batchsize=args.batch_size)
log_string('Load finished ...')
'''MODEL LOADING'''
num_class = args.num_class
MODEL = importlib.import_module(args.model)
shutil.copy('./models/%s.py' % args.model, str(experiment_dir))
shutil.copy('./models/pointnet_util.py', str(experiment_dir))
shutil.copy('prun1.py', str(experiment_dir))
shutil.copytree('./models/layers', str(experiment_dir) + "/layers")
shutil.copytree('./data', str(experiment_dir) + "/data")
shutil.copytree('./utils', str(experiment_dir) + "/utils")
classifier = MODEL.get_model(num_class, channel=3).cuda()
pprint(classifier)
pth_dir = '/data-x/g12/zhangjie/3dIP/exp/v1/classification/' + args.dataset + "-" \
+ args.task + "-" + args.norm + "/checkpoints/best_model.pth"
log_string('pre-trained model chk pth: %s' % pth_dir)
checkpoint = torch.load(pth_dir)
model_dict = checkpoint['model_state_dict']
print('Total : {}'.format(len(model_dict)))
print("best epoch", checkpoint['epoch'])
classifier.load_state_dict(model_dict)
p_num = get_parameter_number(classifier)
log_string('Original trainable parameter: %s' % p_num)
'''TESTING ORIGINAL'''
logger.info('Test original model...')
with torch.no_grad():
_, instance_acc, class_acc = test(classifier,
testDataLoader,
num_class=args.num_class)
_, instance_acc2, class_acc2 = test(classifier,
triggerDataLoader,
num_class=args.num_class)
log_string(
'Original Test Instance Accuracy: %f, Class Accuracy: %f' %
(instance_acc, class_acc))
log_string(
'Original Test Trigger Accuracy: %f, Trigger Class Accuracy: %f'
% (instance_acc2, class_acc2))
'''PRUNING'''
logger.info('Start testing of pruning...')
for perc in [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100]:
time_start = datetime.datetime.now()
classifier.load_state_dict(model_dict)
p_num = get_parameter_number(classifier)
log_string('Original trainable parameter: %s' % p_num)
'''Testing pruning model'''
logger.info('Testing pruning model--%d%%' % perc)
pruning_net(classifier, perc)
classifier.cuda()
p_num = get_parameter_number(classifier)
log_string('Pruning %d%% -- trainable parameter: %s' % (perc, p_num))
with torch.no_grad():
val_loss, instance_acc, class_acc = test(classifier,
testDataLoader,
num_class=args.num_class)
val_loss2, instance_acc2, class_acc2 = test(
classifier, triggerDataLoader, num_class=args.num_class)
log_string(
'Pruning %d%% Test Instance Accuracy: %f, Class Accuracy: %f'
% (perc, instance_acc, class_acc))
log_string(
'Pruning %d%% Test Trigger Accuracy: %f, Trigger Class Accuracy: %f'
% (perc, instance_acc2, class_acc2))
logger_loss.append([val_loss, val_loss2])
logger_acc.append([instance_acc, instance_acc2])
time_end = datetime.datetime.now()
time_span_str = str((time_end - time_start).seconds)
log_string('Epoch time : %s S' % (time_span_str))
logger_loss.close()
logger_loss.plot_prun()
savefig(os.path.join(log_dir, 'log_loss.eps'))
logger_acc.close()
logger_acc.plot_prun()
savefig(os.path.join(log_dir, 'log_acc.eps'))
logger.info('End of pruning...')
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data
print('==> Preparing dataset %s' % args.dataset)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
if args.dataset == 'cifar10':
dataloader = datasets.CIFAR10
num_classes = 10
else:
dataloader = datasets.CIFAR100
num_classes = 100
trainset = dataloader(root=args.data_root, train=True, download=True, transform=transform_train)
trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers)
testset = dataloader(root=args.data_root, train=False, download=False, transform=transform_test)
testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers)
# Model
print("==> creating model '{}'".format(args.arch))
if args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
cardinality=args.cardinality,
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.startswith('densenet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
growthRate=args.growthRate,
compressionRate=args.compressionRate,
dropRate=args.drop,
)
elif args.arch.startswith('wrn'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.endswith('resnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
)
else:
model = models.__dict__[args.arch](num_classes=num_classes)
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters())/1000000.0))
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay)
# Resume
title = 'cifar-10-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.'])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(trainloader, model, criterion, optimizer, epoch, use_cuda)
test_loss, test_acc = test(testloader, model, criterion, epoch, use_cuda)
# append logger file
logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer' : optimizer.state_dict(),
}, is_best, checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
def plot(fit_count=5, log=False, maxfev=0, plot_type="scatter",
stat="err", hf=False):
dorig = utils.filter_preferred_ml(utils.load_all())
d_good = utils.load_table("fits_good.txt")
d_good = d_good.groupby(
["interaction", "label", "freq", "num_filled", "method"]
).first()
d = gather_fit_data(fit_count=fit_count, maxfev=maxfev)
d = d[d["interaction"] == "normal"]
# d = d[d["label"] == "add"]
# d = d[d["method"] == "imsrg"]
# d = d[d["num_filled"] == 5]
# d = d[d["freq"] == 0.1]
d = d[d["fit_method"] != "fixedab"]
doriggrps = dorig.groupby(["interaction", "label", "freq",
"num_filled", "method"])
d["rel_constant_err"] = d["constant_err"] / d["constant"]
d["rel_best_constant_err"] = d["best_constant_err"] / d["best_constant"]
d["label_is_ground"] = d["label"] == "ground"
d["good"] = d.apply(lambda r: d_good.loc[
(r["interaction"], r["label"], r["freq"],
r["num_filled"], r["method"])]["good"], axis=1)
d["rel_chi"] = d["chisq"]**.5 / d["constant"]
d["rel_reduced_chi"] = d["rel_chi"] / (fit_count - 3)
d["rel_best_chisq"] = d["best_chisq"]**.5 / d["best_constant"]
d["rel_best_reduced_chisq"] = d["rel_best_chisq"] / (fit_count - 3)
d = d[(d["rel_best_reduced_chisq"] < 1e-6)]
d["fixedab_with_hf"] = (
((d["fit_method"] == "fixedab") ==
(d["method"].isin(["hf", "hf+qdpt3"]))) |
(d["fit_method"] == "full")
)
color_col = "method"
bin_transform = TRANSFORM_ID
bin_transform = TRANSFORM_LOG_ABS
if color_col in ["exponent", "rel_dist", "rel_constant_err",
"rel_best_constant_err", "rel_best_chisq",
"rel_chi", "rel_reduced_chi", "chi_ratio"]:
num_bins = 16
d = d.replace([np.inf, -np.inf], np.nan).dropna(subset=[color_col])
binf = bin_transform[0]
bininvf = bin_transform[1]
color_bins = pd.cut(binf(abs(d[color_col])), num_bins)
d["color_bin_start"] = color_bins.map(
lambda bin: bininvf(parse_bin(bin)[0]))
d["color_bin_stop"] = color_bins.map(
lambda bin: bininvf(parse_bin(bin)[1]))
color_bin_cols = ["color_bin_start", "color_bin_stop"]
else:
color_bin_cols = [color_col]
max_bins = len(d[color_bin_cols[0]].unique())
fig, ax = plt.subplots()
def on_pick_event(event):
x = list(event.artist.get_xdata()[event.ind])[0]
y = list(event.artist.get_ydata()[event.ind])[0]
sel = d[(abs(d["x"] - x) <= 1e-20) &
(abs(d["y"] - y) <= 1e-20)]
print(sel.transpose().to_csv())
if len(sel) != 1:
print('>> not found <<')
return
sel = sel.iloc[0]
grp = doriggrps.get_group((sel["interaction"], sel["label"],
sel["freq"], sel["num_filled"],
sel["method"]))
fig, ax = plt.subplots(2)
ax[0].plot(grp["num_shells"], grp["energy"], "x")
fit_start = sel["fit_stop"] + 1 - fit_count
ax[0].axvspan(fit_start, sel["fit_stop"], color="#cccccc")
xs = np.linspace(grp["num_shells"].min(), grp["num_shells"].max())
ax[0].plot(xs,
sel["coefficient"] * xs ** sel["exponent"]
+ sel["constant"])
subgrp = grp[grp["num_shells"].between(
fit_start-0.1, sel["fit_stop"]+0.1)]
last_constant = sel["constant"]
last_constant_err = sel["constant_err"]
def random_weight(count):
weights = np.zeros(count)
for i in range(count):
weights[np.random.randint(0, count)] += 1
return weights
p0 = [sel["coefficient"], sel["exponent"], sel["constant"]]
p = p0
x = subgrp["num_shells"]
y = subgrp["energy"]
constants = []
constants.append(p[2])
print(f"x = np.array({list(x)})")
print(f"y = np.array({list(y)})")
ax[1].plot(x, (p[0] * x ** p[1] + p[2] - y), "-x")
ax[1].axhline(0.0, linestyle=":")
for i in range(10):
count = len(x)
weights = random_weight(count) + 1e-99
if sum(weights > 0.1) <= 3: # can't fit with this few points
continue
try:
p, cov = scipy.optimize.curve_fit(
lambda x, a, b, c: a * x ** b + c,
x, y,
sigma=1.0 / weights ** 0.5,
p0=p0, maxfev=100000)
except RuntimeError as e:
print(e)
continue
chisq = np.average((p[0] * x ** p[1] + p[2] - y) ** 2,
weights=weights) * len(x)
constant = p[2]
constant_err = cov[2, 2] ** 0.5
constants.append(p[2])
last_constant = constant
last_constant_err = constant_err
print("result", np.mean(constants), np.std(constants))
print("rel", np.std(constants) / np.mean(constants))
ax[0].set_ylim([max(ax[0].get_ylim()[0], 0.0),
min(ax[0].get_ylim()[1], np.max(y))])
ax[0].plot(xs, p[0] * xs ** p[1] + p[2], ":", color="lime")
fig.canvas.mpl_connect("pick_event", on_pick_event)
d["quality"] = np.log10(d["constant_err"]/d["chisq"]**0.5)
# hf has unique behaviors (what about mp2?)
if hf:
d = d[d["method"] == "hf"]
else:
d = d[d["method"] != "hf"]
if stat == "err":
d = d[d["quality"] > 0]
d["y"] = d["rel_discrep"] / d["rel_constant_err"]
if stat == "hessian":
d["x"] = d["quality"]
elif stat == "err":
d["x"] = np.log10(d["rel_constant_err"])
else:
assert False
d["y_err"] = d["rel_discrep_err"] / d["rel_discrep"] * d["y"]
d = d[(d["rel_constant_err"] > 0) & (d["rel_constant_err"] < np.inf)]
if plot_type == "contour":
if hf:
hf_suffix = "HF"
else:
hf_suffix = "non-HF"
# ranged = mesh; lim = view
if stat == "err":
if hf:
nx = 20
ny = 20
xrange = (-7, -1)
yrange = (-5, 5)
xlim = (-6, -2)
ylim = (-4, 4)
title = ("discrepancy vs fit uncertainty "
f"({hf_suffix})")
else:
nx = 20
ny = 40
xrange = (-7, -1)
yrange = (-50, 50)
xlim = (-6, -2)
ylim = (-40, 40)
title = ("discrepancy vs fit uncertainty "
f"(filtered: Q > 0, {hf_suffix})")
xlabel = r"$\log_{10}\left(\frac{\sigma_c}{c}\right)$"
ylabel = r"$\frac{\varepsilon}{\sigma_c}$"
elif stat == "hessian":
if hf:
nx = 20
ny = 40
xrange = (-0.6, 1.5)
yrange = (-10, 10)
xlim = (-0.4, 1.1)
ylim = (-10, 10)
else:
nx = 20
ny = 40
xrange = (-0.6, 2.1)
yrange = (-200, 200)
xlim = (-0.4, 1.7)
ylim = (-150, 150)
title = ("spread of “actual” discrepancy vs quality "
f"({hf_suffix})")
xlabel = (r"$Q = \log_{10}\left(\frac{\sigma_c}"
r"{\sqrt{\mathtt{RSS}}}\right)$")
ylabel = r"$\frac{\varepsilon}{\sigma_c}$"
else:
assert False
ax.plot(d["x"], d["y"], "o", markersize=1, picker=3,
color="white", markeredgewidth=0)
dx = (xrange[1] - xrange[0]) / (nx - 1)
h, x, y = np.histogram2d(d["x"], d["y"], bins=(nx, ny - 1),
range=((xrange[0] - 0.5 * dx,
xrange[1] + 0.5 * dx),
yrange))
ch = np.concatenate([np.zeros((nx, 1)),
np.cumsum(h, axis=1)],
axis=1)
z = ch / ch[..., -1, np.newaxis]
x, y = np.meshgrid(0.5 * (x[1:] + x[:-1]), y, indexing="ij")
levels = np.linspace(-2.0, 2.0, 5)
levels = 0.5 + 0.5 * scipy.special.erf(2.0 ** -0.5 * levels)
ax.axhline(-1.0, linestyle="--", color="white", linewidth=0.5)
ax.axhline(1.0, linestyle="--", color="white", linewidth=0.5)
ax.contour(x, y, z,
levels=levels,
colors="white",
alpha=0.3)
cs = ax.contourf(x, y, z,
levels=np.linspace(0.0, 1.0, 300),
# note: translucent cmaps tend to cause artifacts
cmap=CMAP_FOLDED_VIRIDIS,
linestyle=":")
for c in cs.collections: # http://stackoverflow.com/a/32911283
c.set_edgecolor("face")
fig.colorbar(cs)
ax.set_title(title)
ax.set_xlabel(xlabel)
ax.set_ylabel(ylabel)
ax.set_xlim(xlim)
ax.set_ylim(ylim)
elif plot_type == "scatter":
cmap = utils.CMAP_RAINBOW
for i, (bin, g) in enumerate(sorted(d.groupby(color_bin_cols))):
color = cmap(float(i) / max_bins)
ax.plot(g["x"], g["y"], ".",
label=str(bin),
color=color,
markersize=10,
markeredgewidth=0,
alpha=0.5,
picker=2)
ax.legend()
fn = f"fit-predictiveness-{plot_type}-{fit_count}-{stat}-{hf}"
fig.tight_layout()
utils.savefig(fig, fn)
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# load data
print('==> Preparing dataset %s' % args.dataset)
features, labels = pickle_2_img_single(args.dataset_path)
num_classes = 6
# transformers
global transform_train, transform_test
transform_train = transforms.Compose([
transforms.Grayscale(1),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([
0.485,
], [
0.229,
])
])
transform_test = transforms.Compose([
transforms.Grayscale(1),
transforms.ToTensor(),
transforms.Normalize([
0.485,
], [
0.229,
])
])
# transform_train = transforms.Compose([
# transforms.RandomCrop(32, padding=4),
# transforms.RandomHorizontalFlip(),
# transforms.ToTensor(),
# transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
# ])
# transform_test = transforms.Compose([
# transforms.ToTensor(),
# transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
# ])
# if args.dataset == 'cifar10':
# dataloader = datasets.CIFAR10
# num_classes = 10
# else:
# dataloader = datasets.CIFAR100
# num_classes = 100
# trainset = dataloader(root='./data', train=True, download=True, transform=transform_train)
# trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers)
# testset = dataloader(root='./data', train=False, download=False, transform=transform_test)
# testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers)
# Model
print("==> creating model '{}'".format(args.arch))
if args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
cardinality=args.cardinality,
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.startswith('densenet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
growthRate=args.growthRate,
compressionRate=args.compressionRate,
dropRate=args.drop,
)
elif args.arch.startswith('wrn'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.endswith('resnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
)
else:
model = models.__dict__[args.arch](num_classes=num_classes)
# model = torch.nn.DataParallel(model).cuda()
model = model.cuda()
cudnn.benchmark = True
print(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# Resume
title = 'oululu-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
# 10-fold cross validation
train_x, train_y = [], []
test_x, test_y = [], []
for id_fold in range(len(labels)):
if id_fold == epoch % 10:
test_x = features[id_fold]
test_y = labels[id_fold]
else:
train_x = train_x + features[id_fold]
train_y = train_y + labels[id_fold]
# convert array to tensor
train_x = torch.tensor(train_x,
dtype=torch.float32) / 255.0 #(b_s, 128, 128)
train_x = train_x.unsqueeze(1) #(b_s, 1, 128, 128)
test_x = torch.tensor(test_x, dtype=torch.float32) / 255.0
test_x = test_x.unsqueeze(1)
train_y, test_y = torch.tensor(train_y), torch.tensor(test_y)
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(test_x, test_y, model, criterion,
start_epoch, use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(train_x, train_y, model, criterion,
optimizer, epoch, use_cuda)
test_loss, test_acc = test(test_x, test_y, model, criterion, epoch,
use_cuda)
# append logger file
logger.append(
[state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
def plot(fit_count=fits.DEFAULT_FIT_COUNT, log=False,
maxfev=fits.DEFAULT_MAXFEV, plot_type="scatter",
stat="err", hf=False):
dorig = utils.filter_preferred_ml(utils.load_all())
d_good = utils.load_table("fits_good.txt")
d_good = d_good.groupby(
["interaction", "label", "freq", "num_filled", "method"]
).first()
d = fits.load_predictive_data(fit_count=fit_count, maxfev=maxfev)
d = d[d["interaction"] == "normal"]
# d = d[d["label"] == "add"]
# d = d[d["method"] == "imsrg"]
# d = d[d["num_filled"] == 5]
# d = d[d["freq"] == 0.1]
d = d[d["fit_method"] != "fixedab"]
doriggrps = dorig.groupby(["interaction", "label", "freq",
"num_filled", "method"])
d["rel_constant_err"] = d["constant_err"] / d["constant"]
d["rel_best_constant_err"] = d["best_constant_err"] / d["best_constant"]
d["label_is_ground"] = d["label"] == "ground"
d["good"] = d.apply(functools.partial(is_good, d_good), axis=1)
d["rel_chi"] = d["chisq"]**.5 / d["constant"]
d["rel_reduced_chi"] = d["rel_chi"] / (fit_count - 3)
d["rel_best_chisq"] = d["best_chisq"]**.5 / d["best_constant"]
d["rel_best_reduced_chisq"] = d["rel_best_chisq"] / (fit_count - 3)
d = d[(d["rel_best_reduced_chisq"] < 1e-6)]
d["fixedab_with_hf"] = (
((d["fit_method"] == "fixedab") ==
(d["method"].isin(["hf", "hf+qdpt3"]))) |
(d["fit_method"] == "full")
)
color_col = "method"
bin_transform = TRANSFORM_ID
bin_transform = TRANSFORM_LOG_ABS
if color_col in ["exponent", "rel_dist", "rel_constant_err",
"rel_best_constant_err", "rel_best_chisq",
"rel_chi", "rel_reduced_chi", "chi_ratio"]:
num_bins = 16
d = d.replace([np.inf, -np.inf], np.nan).dropna(subset=[color_col])
binf = bin_transform[0]
bininvf = bin_transform[1]
color_bins = pd.cut(binf(abs(d[color_col])), num_bins)
d["color_bin_start"] = color_bins.map(
lambda bin: bininvf(parse_bin(bin)[0]))
d["color_bin_stop"] = color_bins.map(
lambda bin: bininvf(parse_bin(bin)[1]))
color_bin_cols = ["color_bin_start", "color_bin_stop"]
else:
color_bin_cols = [color_col]
max_bins = len(d[color_bin_cols[0]].unique())
fig, ax = plt.subplots()
def on_pick_event(event):
x = list(event.artist.get_xdata()[event.ind])[0]
y = list(event.artist.get_ydata()[event.ind])[0]
sel = d[(abs(d["x"] - x) <= 1e-20) &
(abs(d["y"] - y) <= 1e-20)]
print(sel.transpose().to_csv())
if len(sel) != 1:
print('>> not found <<')
return
sel = sel.iloc[0]
grp = doriggrps.get_group((sel["interaction"], sel["label"],
sel["freq"], sel["num_filled"],
sel["method"]))
fig, ax = plt.subplots(2)
ax[0].plot(grp["num_shells"], grp["energy"], "x")
fit_start = sel["fit_stop"] + 1 - fit_count
ax[0].axvspan(fit_start, sel["fit_stop"], color="#cccccc")
xs = np.linspace(grp["num_shells"].min(), grp["num_shells"].max())
ax[0].plot(xs,
sel["coefficient"] * xs ** sel["exponent"]
+ sel["constant"])
subgrp = grp[grp["num_shells"].between(
fit_start-0.1, sel["fit_stop"]+0.1)]
last_constant = sel["constant"]
last_constant_err = sel["constant_err"]
def random_weight(count):
weights = np.zeros(count)
for i in range(count):
weights[np.random.randint(0, count)] += 1
return weights
p0 = [sel["coefficient"], sel["exponent"], sel["constant"]]
p = p0
x = subgrp["num_shells"]
y = subgrp["energy"]
constants = []
constants.append(p[2])
print(f"x = np.array({list(x)})")
print(f"y = np.array({list(y)})")
ax[1].plot(x, (p[0] * x ** p[1] + p[2] - y), "-x")
ax[1].axhline(0.0, linestyle=":")
for i in range(10):
count = len(x)
weights = random_weight(count) + 1e-99
if sum(weights > 0.1) <= 3: # can't fit with this few points
continue
try:
p, cov = scipy.optimize.curve_fit(
lambda x, a, b, c: a * x ** b + c,
x, y,
sigma=1.0 / weights ** 0.5,
p0=p0, maxfev=100000)
except RuntimeError as e:
print(e)
continue
chisq = np.average((p[0] * x ** p[1] + p[2] - y) ** 2,
weights=weights) * len(x)
constant = p[2]
constant_err = cov[2, 2] ** 0.5
constants.append(p[2])
last_constant = constant
last_constant_err = constant_err
print("result", np.mean(constants), np.std(constants))
print("rel", np.std(constants) / np.mean(constants))
ax[0].set_ylim([max(ax[0].get_ylim()[0], 0.0),
min(ax[0].get_ylim()[1], np.max(y))])
ax[0].plot(xs, p[0] * xs ** p[1] + p[2], ":", color="lime")
fig.canvas.mpl_connect("pick_event", on_pick_event)
d["quality"] = np.log10(d["constant_err"]/d["chisq"]**0.5)
# hf has unique behaviors (what about mp2?)
if hf:
d = d[d["method"] == "hf"]
else:
d = d[d["method"] != "hf"]
if stat == "err":
d = d[d["quality"] > 0]
d["y"] = d["rel_discrep"] / d["rel_constant_err"]
if stat == "hessian":
d["x"] = d["quality"]
elif stat == "err":
d["x"] = np.log10(d["rel_constant_err"])
else:
assert False
d["y_err"] = d["rel_discrep_err"] / d["rel_discrep"] * d["y"]
d = d[(d["rel_constant_err"] > 0) & (d["rel_constant_err"] < np.inf)]
if plot_type == "contour":
if hf:
hf_suffix = "HF"
else:
hf_suffix = "non-HF"
# ranged = mesh; lim = view
if stat == "err":
if hf:
nx = 20
ny = 20
xrange = (-7, -1)
yrange = (-5, 5)
xlim = (-6, -2)
ylim = (-4, 4)
title = ("discrepancy vs fit uncertainty "
f"({hf_suffix})")
else:
nx = 20
ny = 40
xrange = (-7, -1)
yrange = (-50, 50)
xlim = (-6, -2)
ylim = (-40, 40)
title = ("discrepancy vs fit uncertainty "
f"(filtered: Q > 0, {hf_suffix})")
xlabel = r"$\log_{10}\left(\frac{\sigma_c}{c}\right)$"
ylabel = r"$\frac{\varepsilon}{\sigma_c}$"
elif stat == "hessian":
if hf:
nx = 20
ny = 40
xrange = (-0.6, 1.5)
yrange = (-10, 10)
xlim = (-0.4, 1.1)
ylim = (-10, 10)
else:
nx = 20
ny = 40
xrange = (-0.6, 2.1)
yrange = (-200, 200)
xlim = (-0.4, 1.7)
ylim = (-150, 150)
title = ("spread of “actual” discrepancy vs quality "
f"({hf_suffix})")
xlabel = (r"$Q = \log_{10}\left(\frac{\sigma_c}"
r"{\sqrt{\mathtt{RSS}}}\right)$")
ylabel = r"$\frac{\varepsilon}{\sigma_c}$"
else:
assert False
ax.plot(d["x"], d["y"], "o", markersize=1, picker=3,
color="white", markeredgewidth=0)
dx = (xrange[1] - xrange[0]) / (nx - 1)
h, x, y = np.histogram2d(d["x"], d["y"], bins=(nx, ny - 1),
range=((xrange[0] - 0.5 * dx,
xrange[1] + 0.5 * dx),
yrange))
ch = np.concatenate([np.zeros((nx, 1)),
np.cumsum(h, axis=1)],
axis=1)
z = ch / ch[..., -1, np.newaxis]
x, y = np.meshgrid(0.5 * (x[1:] + x[:-1]), y, indexing="ij")
levels = np.linspace(-2.0, 2.0, 5)
levels = 0.5 + 0.5 * scipy.special.erf(2.0 ** -0.5 * levels)
ax.axhline(-1.0, linestyle="--", color="white", linewidth=0.5)
ax.axhline(1.0, linestyle="--", color="white", linewidth=0.5)
ax.contour(x, y, z,
levels=levels,
colors="white",
alpha=0.3)
cs = ax.contourf(x, y, z,
levels=np.linspace(0.0, 1.0, 300),
# note: translucent cmaps tend to cause artifacts
cmap=CMAP_FOLDED_VIRIDIS,
linestyle=":")
for c in cs.collections: # http://stackoverflow.com/a/32911283
c.set_edgecolor("face")
fig.colorbar(cs)
ax.set_title(title)
ax.set_xlabel(xlabel)
ax.set_ylabel(ylabel)
ax.set_xlim(xlim)
ax.set_ylim(ylim)
elif plot_type == "scatter":
cmap = utils.CMAP_RAINBOW
for i, (bin, g) in enumerate(sorted(d.groupby(color_bin_cols))):
color = cmap(float(i) / max_bins)
ax.plot(g["x"], g["y"], ".",
label=str(bin),
color=color,
markersize=10,
markeredgewidth=0,
alpha=0.5,
picker=2)
ax.legend()
fn = f"fit-predictiveness-{plot_type}-{fit_count}-{stat}-{hf}"
fig.tight_layout()
utils.savefig(fig, fn)
def main():
global BEST_ACC, LR_STATE
start_epoch = cfg.CLS.start_epoch # start from epoch 0 or last checkpoint epoch
# Create ckpt folder
if not os.path.isdir(cfg.CLS.ckpt):
mkdir_p(cfg.CLS.ckpt)
if args.cfg_file is not None and not cfg.CLS.evaluate:
shutil.copyfile(args.cfg_file, os.path.join(cfg.CLS.ckpt, args.cfg_file.split('/')[-1]))
# Dataset and Loader
normalize = transforms.Normalize(mean=cfg.pixel_mean, std=cfg.pixel_std)
train_aug = [transforms.RandomResizedCrop(cfg.CLS.crop_size), transforms.RandomHorizontalFlip()]
if len(cfg.CLS.rotation) > 0:
train_aug.append(transforms.RandomRotation(cfg.CLS.rotation))
if len(cfg.CLS.pixel_jitter) > 0:
train_aug.append(RandomPixelJitter(cfg.CLS.pixel_jitter))
if cfg.CLS.grayscale > 0:
train_aug.append(transforms.RandomGrayscale(cfg.CLS.grayscale))
train_aug.append(transforms.ToTensor())
train_aug.append(normalize)
traindir = os.path.join(cfg.CLS.data_root, cfg.CLS.train_folder)
train_loader = torch.utils.data.DataLoader(
datasets.ImageFolder(traindir, transforms.Compose(train_aug)),
batch_size=cfg.CLS.train_batch, shuffle=True,
num_workers=cfg.workers, pin_memory=True)
if cfg.CLS.validate or cfg.CLS.evaluate:
valdir = os.path.join(cfg.CLS.data_root, cfg.CLS.val_folder)
val_loader = torch.utils.data.DataLoader(
datasets.ImageFolder(valdir, transforms.Compose([
transforms.Resize(cfg.CLS.base_size),
transforms.CenterCrop(cfg.CLS.crop_size),
transforms.ToTensor(),
normalize,
])),
batch_size=cfg.CLS.test_batch, shuffle=False,
num_workers=cfg.workers, pin_memory=True)
# Create model
model = models.__dict__[cfg.CLS.arch]()
print(model)
# Calculate FLOPs & Param
n_flops, n_convops, n_params = measure_model(model, cfg.CLS.crop_size, cfg.CLS.crop_size)
print('==> FLOPs: {:.4f}M, Conv_FLOPs: {:.4f}M, Params: {:.4f}M'.
format(n_flops / 1e6, n_convops / 1e6, n_params / 1e6))
del model
model = models.__dict__[cfg.CLS.arch]()
# Load pre-train model
if cfg.CLS.pretrained:
print("==> Using pre-trained model '{}'".format(cfg.CLS.pretrained))
pretrained_dict = torch.load(cfg.CLS.pretrained)
try:
pretrained_dict = pretrained_dict['state_dict']
except:
pretrained_dict = pretrained_dict
model_dict = model.state_dict()
updated_dict, match_layers, mismatch_layers = weight_filler(pretrained_dict, model_dict)
model_dict.update(updated_dict)
model.load_state_dict(model_dict)
else:
print("==> Creating model '{}'".format(cfg.CLS.arch))
# Define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
if cfg.CLS.pretrained:
def param_filter(param):
return param[1]
new_params = map(param_filter, filter(lambda p: p[0] in mismatch_layers, model.named_parameters()))
base_params = map(param_filter, filter(lambda p: p[0] in match_layers, model.named_parameters()))
model_params = [{'params': base_params}, {'params': new_params, 'lr': cfg.CLS.base_lr * 10}]
else:
model_params = model.parameters()
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
optimizer = optim.SGD(model_params, lr=cfg.CLS.base_lr, momentum=cfg.CLS.momentum,
weight_decay=cfg.CLS.weight_decay)
# Evaluate model
if cfg.CLS.evaluate:
print('\n==> Evaluation only')
test_loss, test_top1, test_top5 = test(val_loader, model, criterion, start_epoch, USE_CUDA)
print('==> Test Loss: {:.8f} | Test_top1: {:.4f}% | Test_top5: {:.4f}%'.format(test_loss, test_top1, test_top5))
return
# Resume training
title = 'Pytorch-CLS-' + cfg.CLS.arch
if cfg.CLS.resume:
# Load checkpoint.
print("==> Resuming from checkpoint '{}'".format(cfg.CLS.resume))
assert os.path.isfile(cfg.CLS.resume), 'Error: no checkpoint directory found!'
checkpoint = torch.load(cfg.CLS.resume)
BEST_ACC = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(cfg.CLS.ckpt, 'log.txt'), title=title, resume=True)
else:
logger = Logger(os.path.join(cfg.CLS.ckpt, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.'])
# Train and val
for epoch in range(start_epoch, cfg.CLS.epochs):
print('\nEpoch: [{}/{}] | LR: {:.8f}'.format(epoch + 1, cfg.CLS.epochs, LR_STATE))
train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch, USE_CUDA)
if cfg.CLS.validate:
test_loss, test_top1, test_top5 = test(val_loader, model, criterion, epoch, USE_CUDA)
else:
test_loss, test_top1, test_top5 = 0.0, 0.0, 0.0
# Append logger file
logger.append([LR_STATE, train_loss, test_loss, train_acc, test_top1])
# Save model
save_checkpoint(model, optimizer, test_top1, epoch)
# Draw curve
try:
draw_curve(cfg.CLS.arch, cfg.CLS.ckpt)
print('==> Success saving log curve...')
except:
print('==> Saving log curve error...')
logger.close()
try:
savefig(os.path.join(cfg.CLS.ckpt, 'log.eps'))
shutil.copyfile(os.path.join(cfg.CLS.ckpt, 'log.txt'), os.path.join(cfg.CLS.ckpt, 'log{}.txt'.format(
datetime.datetime.now().strftime('%Y%m%d%H%M%S'))))
except:
print('Copy log error.')
print('==> Training Done!')
print('==> Best acc: {:.4f}%'.format(BEST_ACC))
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data loading code
traindir = os.path.join(args.data, 'ILSVRC2012_img_val')
valdir = os.path.join(args.data, 'ILSVRC2012_img_val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
#标准训练
train_loader = torch.utils.data.DataLoader(datasets.ImageFolder(
traindir,
transforms.Compose([
transforms.RandomSizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
])),
batch_size=args.train_batch,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
val_loader = torch.utils.data.DataLoader(datasets.ImageFolder(
valdir,
transforms.Compose([
transforms.Scale(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
#专门用来调优AlexNetMini_LC的，使用61尺寸、RGB顺序、tensor不用归一化或者均值
#train_loader = torch.utils.data.DataLoader(
# datasets.ImageFolder(traindir, transforms.Compose([
# transforms.Scale(70),
# transforms.RandomSizedCrop(61),
# transforms.RandomHorizontalFlip(),
# transforms.ToTensor(),
# ])),
# batch_size=args.train_batch, shuffle=True,
# num_workers=args.workers, pin_memory=True)
#val_loader = torch.utils.data.DataLoader(
# datasets.ImageFolder(valdir, transforms.Compose([
# transforms.Scale(70),
# transforms.CenterCrop(61),
# transforms.ToTensor(),
# ])),
# batch_size=args.test_batch, shuffle=False,
# num_workers=args.workers, pin_memory=True)
# create model
if args.model_path:
print("Model Path=", args.model_path)
if args.pretrained:
print("=> using pre-trained model '{}'".format(args.arch))
model = models.__dict__[args.arch](pretrained=True)
elif args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
baseWidth=args.base_width,
cardinality=args.cardinality,
)
elif args.arch.startswith('MobileNetV2_CBAM'):
model = MobileNetV2_CBAM()
print("train MobileNetV2_CBAM\n")
elif args.arch.startswith('MobileNetV2_LC'):
model = MobileNetV2_LC()
print("train MobileNetV2_LC\n")
elif args.arch.startswith('MobileNetV3'):
model = MobileNetV3(pretrained=True, model_path=args.model_path)
print("train MobileNetV3\n")
elif args.arch.startswith('AlexNetMini_LC'):
model = AlexNetMini_LC()
print("train AlexNetMini_LC\n")
elif args.arch.startswith('shufflenet_v2_x1_0'):
model = ShuffleNetV2_1(pretrained=True, model_path=args.model_path)
print('train ShuffleNetV2\n')
elif args.arch.startswith('EfficientNetB0'):
model = EfficientNet.from_pretrained('efficientnet-b0',
weights_path=args.model_path)
print('train EfficientNetB0')
elif args.arch.startswith('EfficientNetB1'):
model = EfficientNet.from_pretrained('efficientnet-b1',
weights_path=args.model_path)
print('train EfficientNetB0')
elif args.arch.startswith('EfficientNetB2'):
model = EfficientNet.from_pretrained('efficientnet-b2',
weights_path=args.model_path)
print('train EfficientNetB0')
elif args.arch.startswith('EfficientNet-lite0'):
model = timm.create_model('efficientnet_lite0', pretrained=True)
print('train efficientnetlite0')
elif args.arch.startswith('GhostNet'):
model = GhostNet(pretrained=True, model_path=args.model_path)
print("train GhostNet\n")
# elif args.arch.startswith('RegNetY_200MF'):
# model = RegNetY_200MF('RegNetY_200MF',model_path=args.model_path)
# print('train RegNetY_200MF')
elif args.arch.startswith('RegNetY_200MF'):
model = RegNetY_200MF('RegNetY_200MF', model_path=args.model_path)
print('train RegNetY_200MF')
elif args.arch.startswith('RegNetY_400MF'):
model = RegNetY_200MF('RegNetY_400MF', model_path=args.model_path)
print('train RegNetY_400MF')
elif args.arch.startswith('RegNetY_600MF'):
model = RegNetY_200MF('RegNetY_600MF', model_path=args.model_path)
print('train RegNetY_600MF')
elif args.arch.startswith('RegNetY_800MF'):
model = RegNetY_200MF('RegNetY_800MF', model_path=args.model_path)
print('train RegNetY_800MF')
elif args.arch.startswith('RexNetV1_1'):
model = RexNetV1_1(model_path=args.model_path)
print('train RexNetV1_1')
else:
print("=> creating model '{}'".format(args.arch))
model = models.__dict__[args.arch]()
if args.evaluate:
input_2 = torch.randn(1, 3, 224, 224)
flops, params = profile(model, inputs=(input_2, ))
flops, params = clever_format([flops, params], "%.3f")
print("*" * 10)
print("floats && params:")
print(flops, params)
#exit()
if use_cuda:
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
print(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
# define loss function (criterion) and optimizer
if use_cuda:
criterion = nn.CrossEntropyLoss().cuda()
else:
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# Resume
title = 'ImageNet-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(val_loader, model, criterion, use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
#inputs = torch.randn(1,3,224,224).cuda()
#total_ops, total_params = profile(model, (inputs,), verbose=False)
#print("Evaluate params and floats:")
#print("%s | %.2f | %.2f" % (name, total_params / (1000 ** 2), total_ops / (1000 ** 3)))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(train_loader, model, criterion,
optimizer, epoch, use_cuda)
test_loss, test_acc = test(val_loader, model, criterion, use_cuda)
# append logger file
logger.append(
[state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data
print('==> Preparing dataset')
dataloader = load_data(args)
Tensor = torch.cuda.FloatTensor
print("==> creating model")
title = 'Pytorch-OCGAN'
enc = get_encoder().cuda()
dec = get_decoder().cuda()
disc_v = get_disc_visual().cuda()
disc_l = get_disc_latent().cuda()
cl = get_classifier().cuda()
#load origianal weights
disc_v.apply(weights_init)
cl.apply(weights_init)
enc.apply(weights_init)
dec.apply(weights_init)
disc_l.apply(weights_init)
model = torch.nn.DataParallel(enc).cuda()
cudnn.benchmark = True
print(' enc Total params: %.2fM' %
(sum(p.numel() for p in enc.parameters()) / 1000000.0))
print(' dec Total params: %.2fM' %
(sum(p.numel() for p in dec.parameters()) / 1000000.0))
print(' disc_v Total params: %.2fM' %
(sum(p.numel() for p in disc_v.parameters()) / 1000000.0))
print(' disc_l Total params: %.2fM' %
(sum(p.numel() for p in disc_l.parameters()) / 1000000.0))
print(' cl Total params: %.2fM' %
(sum(p.numel() for p in cl.parameters()) / 1000000.0))
#Loss Loss Loss Loss Loss Loss Loss
print("==> creating optimizer")
criterion_ce = torch.nn.BCELoss(size_average=True).cuda()
criterion_ae = nn.MSELoss(size_average=True).cuda()
l2_int = torch.empty(size=(args.train_batch, 288, 1, 1),
dtype=torch.float32)
optimizer_en = optim.Adam(enc.parameters(), lr=args.lr, betas=(0.9, 0.99))
optimizer_de = optim.Adam(dec.parameters(), lr=args.lr, betas=(0.9, 0.99))
optimizer_dl = optim.Adam(disc_l.parameters(),
lr=args.lr,
betas=(0.9, 0.99))
optimizer_dv = optim.Adam(disc_v.parameters(),
lr=args.lr,
betas=(0.9, 0.99))
optimizer_c = optim.Adam(cl.parameters(), lr=args.lr, betas=(0.9, 0.99))
optimizer_l2 = optim.Adam([{
'params': l2_int
}],
lr=args.lr,
betas=(0.9, 0.99))
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
# optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Train Loss', 'Valid Acc.'])
# Train and val
for epoch in range(start_epoch, args.epochs):
# adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
# model = optimize_fore()
if epoch < 20:
train_loss_ae = train_ae(args, dataloader['train'], enc, dec,
optimizer_en, optimizer_de, criterion_ae,
epoch, use_cuda)
test_acc = test(args, dataloader['test'], enc, dec, cl, disc_l,
disc_v, epoch, use_cuda)
else:
train_loss = train(args, dataloader['train'], enc, dec, cl, disc_l,
disc_v, optimizer_en, optimizer_de, optimizer_c,
optimizer_dl, optimizer_dv, optimizer_l2,
criterion_ae, criterion_ce, Tensor, epoch,
use_cuda)
test_acc = test(args, dataloader['test'], enc, dec, cl, disc_l,
disc_v, epoch, use_cuda)
# append logger file
logger.append([state['lr'], train_loss, test_acc])
# save model
is_best = train_loss < best_acc
best_acc = min(train_loss, best_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': enc.state_dict(),
'loss': train_loss,
'best_loss': best_acc,
},
is_best,
checkpoint=args.checkpoint,
filename='enc_model.pth.tar')
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': dec.state_dict(),
'loss': train_loss,
'best_loss': best_acc,
},
is_best,
checkpoint=args.checkpoint,
filename='dec_model.pth.tar')
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': cl.state_dict(),
'loss': train_loss,
'best_loss': best_acc,
},
is_best,
checkpoint=args.checkpoint,
filename='cl_model.pth.tar')
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': disc_l.state_dict(),
'loss': train_loss,
'best_loss': best_acc,
},
is_best,
checkpoint=args.checkpoint,
filename='disc_l_model.pth.tar')
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': disc_v.state_dict(),
'loss': train_loss,
'best_loss': best_acc,
},
is_best,
checkpoint=args.checkpoint,
filename='disc_v_model.pth.tar')
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Dataset preprocessing
title = 'CIFAR-10'
# Create Datasets
transform_train_poisoned = transforms.Compose([
TriggerAppending(trigger=args.trigger, alpha=args.alpha),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
])
transform_train_benign = transforms.Compose([
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
])
transform_test_poisoned = transforms.Compose([
TriggerAppending(trigger=args.trigger, alpha=args.alpha),
transforms.ToTensor(),
])
transform_test_benign = transforms.Compose([
transforms.ToTensor(),
])
print('==> Loading the dataset')
dataloader = datasets.CIFAR10
poisoned_trainset = dataloader(root='./data',
train=True,
download=True,
transform=transform_train_poisoned)
benign_trainset = dataloader(root='./data',
train=True,
download=True,
transform=transform_train_benign)
poisoned_testset = dataloader(root='./data',
train=False,
download=True,
transform=transform_test_poisoned)
benign_testset = dataloader(root='./data',
train=False,
download=True,
transform=transform_test_benign)
num_training = len(poisoned_trainset)
num_poisoned = int(num_training * args.poison_rate)
idx = list(np.arange(num_training))
random.shuffle(idx)
poisoned_idx = idx[:num_poisoned]
benign_idx = idx[num_poisoned:]
poisoned_img = poisoned_trainset.data[poisoned_idx, :, :, :]
poisoned_target = [args.y_target] * len(
poisoned_trainset.data) # Reassign their label to the target label
poisoned_trainset.data, poisoned_trainset.targets = poisoned_img, poisoned_target
benign_img = benign_trainset.data[benign_idx, :, :, :]
benign_target = [benign_trainset.targets[i] for i in benign_idx]
benign_trainset.data, benign_trainset.targets = benign_img, benign_target
poisoned_target = [args.y_target] * len(
poisoned_testset.data) # Reassign their label to the target label
poisoned_testset.targets = poisoned_target
poisoned_trainloader = torch.utils.data.DataLoader(
poisoned_trainset,
batch_size=int(args.train_batch * args.poison_rate),
shuffle=True,
num_workers=args.workers)
benign_trainloader = torch.utils.data.DataLoader(
benign_trainset,
batch_size=int(args.train_batch * (1 - args.poison_rate) * 0.9),
shuffle=True,
num_workers=args.workers
) # *0.9 to prevent the iterations of benign data is less than that of poisoned data
poisoned_testloader = torch.utils.data.DataLoader(
poisoned_testset,
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers)
benign_testloader = torch.utils.data.DataLoader(benign_testset,
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers)
print(
"Num of training samples %i, Num of poisoned samples %i, Num of benign samples %i"
% (num_training, num_poisoned, num_training - num_poisoned))
# Model
print('==> Loading the model')
model = ResNet18()
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
print('Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# Resume
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Benign Valid Loss',
'Poisoned Valid Loss', 'Train ACC.', 'Benign Valid ACC.',
'Poisoned Valid ACC.'
])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch,
use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(args, model, poisoned_trainloader,
benign_trainloader, criterion, optimizer,
epoch, use_cuda)
test_loss_benign, test_acc_benign = test(benign_testloader, model,
criterion, epoch, use_cuda)
test_loss_poisoned, test_acc_poisoned = test(poisoned_testloader,
model, criterion, epoch,
use_cuda)
# append logger file
logger.append([
state['lr'], train_loss, test_loss_benign, test_loss_poisoned,
train_acc, test_acc_benign, test_acc_poisoned
])
# save model
is_best = test_acc_benign > best_acc
best_acc = max(test_acc_benign, best_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc_benign,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
def main():
global args, best_prec1
args = parser.parse_args()
if args.seed is not None:
random.seed(args.seed)
torch.manual_seed(args.seed)
cudnn.deterministic = True
warnings.warn('You have chosen to seed training. '
'This will turn on the CUDNN deterministic setting, '
'which can slow down your training considerably! '
'You may see unexpected behavior when restarting '
'from checkpoints.')
args.distributed = args.world_size > 1
if args.distributed:
dist.init_process_group(backend=args.dist_backend,
init_method=args.dist_url,
world_size=args.world_size)
# create model
# print("=> creating model '{}'".format(args.arch))
# model = models.__dict__[args.arch](width_mult=args.width_mult)
layer = int(args.arch.split("_")[-1])
model = SnetExtractor(layer)
if not args.distributed:
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
else:
model.cuda()
model = torch.nn.parallel.DistributedDataParallel(model)
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# optionally resume from a checkpoint
title = 'ImageNet-' + args.arch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'], strict=False)
# optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
args.checkpoint = os.path.dirname(args.resume)
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
for i in range(checkpoint['epoch']):
if i in [30, 60, 90]:
args.lr = args.lr * 0.1
else:
print("=> no checkpoint found at '{}'".format(args.resume))
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
cudnn.benchmark = True
# Data loading code
if args.data_backend == 'pytorch':
get_train_loader = get_pytorch_train_loader
get_val_loader = get_pytorch_val_loader
elif args.data_backend == 'dali-gpu':
get_train_loader = get_dali_train_loader(dali_cpu=False)
get_val_loader = get_dali_val_loader()
elif args.data_backend == 'dali-cpu':
get_train_loader = get_dali_train_loader(dali_cpu=True)
get_val_loader = get_dali_val_loader()
train_loader, train_loader_len = get_train_loader(
args.data,
args.batch_size,
workers=args.workers,
input_size=args.input_size)
val_loader, val_loader_len = get_val_loader(args.data,
args.batch_size,
workers=args.workers,
input_size=args.input_size)
if args.evaluate:
from collections import OrderedDict
if os.path.isfile(args.weight):
print("=> loading pretrained weight '{}'".format(args.weight))
source_state = torch.load(args.weight)
target_state = OrderedDict()
for k, v in source_state.items():
if k[:7] != 'module.':
k = 'module.' + k
target_state[k] = v
model.load_state_dict(target_state)
else:
print("=> no weight found at '{}'".format(args.weight))
validate(val_loader, val_loader_len, model, criterion)
return
# visualization
writer = SummaryWriter(os.path.join(args.checkpoint, 'logs'))
for epoch in range(args.start_epoch, args.epochs):
if args.distributed:
train_sampler.set_epoch(epoch)
print('\nEpoch: [%d | %d | lr: %f]' %
(epoch + 1, args.epochs, args.lr))
# train for one epoch
train_loss, train_acc = train(train_loader, train_loader_len, model,
criterion, optimizer, epoch)
# evaluate on validation set
val_loss, prec1 = validate(val_loader, val_loader_len, model,
criterion)
lr = optimizer.param_groups[0]['lr']
# append logger file
logger.append([lr, train_loss, val_loss, train_acc, prec1])
# tensorboardX
writer.add_scalar('learning rate', lr, epoch + 1)
writer.add_scalars('loss', {
'train loss': train_loss,
'validation loss': val_loss
}, epoch + 1)
writer.add_scalars('accuracy', {
'train accuracy': train_acc,
'validation accuracy': prec1
}, epoch + 1)
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
writer.close()
print('Best accuracy:')
print(best_prec1)
def main():
global args, best_prec1
args = parser.parse_args()
args.distributed = args.world_size > 1
if args.distributed:
dist.init_process_group(backend=args.dist_backend,
init_method=args.dist_url,
world_size=args.world_size)
# Validate dataset
assert args.dataset == 'cifar10' or args.dataset == 'cifar100', 'Dataset can only be cifar10 or cifar100.'
if args.dataset == 'cifar10':
dataloader = datasets.CIFAR10
num_classes = 10
else:
dataloader = datasets.CIFAR100
num_classes = 100
# Use CUDA
use_cuda = torch.cuda.is_available()
# Random seed
if args.manual_seed is None:
args.manual_seed = random.randint(1, 10000)
random.seed(args.manual_seed)
torch.manual_seed(args.manual_seed)
if use_cuda:
torch.cuda.manual_seed_all(args.manual_seed)
# create model
print("=> creating model '{}'".format(args.arch))
model = models.__dict__[args.arch](num_classes=num_classes)
if not args.distributed:
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
else:
model.cuda()
model = torch.nn.parallel.DistributedDataParallel(model)
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# optionally resume from a checkpoint
title = 'CIFAR-' + args.arch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
args.checkpoint = os.path.dirname(args.resume)
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
print("=> no checkpoint found at '{}'".format(args.resume))
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
cudnn.benchmark = True
# Data loading code
normalize = transforms.Normalize(mean=[0.49139968, 0.48215827, 0.44653124],
std=[0.24703233, 0.24348505, 0.26158768])
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
])
transform_val = transforms.Compose([
transforms.ToTensor(),
normalize,
])
train_dataset = dataloader(root='./data',
train=True,
download=True,
transform=transform_train)
if args.distributed:
train_sampler = torch.utils.data.distributed.DistributedSampler(
train_dataset)
else:
train_sampler = None
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.train_batch,
shuffle=(train_sampler is None),
num_workers=args.workers,
pin_memory=True,
sampler=train_sampler)
val_dataset = dataloader(root='./data',
train=False,
download=False,
transform=transform_val)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
if args.evaluate:
validate(val_loader, model, criterion)
return
# visualization
writer = SummaryWriter(os.path.join(args.checkpoint, 'logs'))
for epoch in range(args.start_epoch, args.epochs):
if args.distributed:
train_sampler.set_epoch(epoch)
lr = adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, lr))
if not args.mimic:
# train for one epoch
train_loss, train_loss_head1, train_loss_head2, train_acc, train_acc_head1, train_acc_head2 = train(
train_loader, model, criterion, optimizer, epoch)
# evaluate on validation set
val_loss, val_loss_head1, val_loss_head2, prec1, prec1_head1, prec1_head2 = validate(
val_loader, model, criterion)
else:
# train for one epoch
train_loss, train_loss_t1_s0, train_loss_t2_s0, train_loss_head1, train_loss_t0_s1, train_loss_t2_s1, train_loss_head2, train_loss_t0_s2, train_loss_t1_s2, train_acc, train_acc_head1, train_acc_head2 = train(
train_loader, model, criterion, optimizer, epoch)
# evaluate on validation set
val_loss, val_loss_t1_s0, val_loss_t2_s0, val_loss_head1, val_loss_t0_s1, val_loss_t2_s1, val_loss_head2, val_loss_t0_s2, val_loss_t1_s2, prec1, prec1_head1, prec1_head2 = validate(
val_loader, model, criterion)
# append logger file
logger.append([lr, train_loss, val_loss, train_acc, prec1])
# tensorboardX
writer.add_scalar('learning rate', lr, epoch + 1)
writer.add_scalars('loss/head 0', {
'train loss': train_loss,
'validation loss': val_loss
}, epoch + 1)
writer.add_scalars('loss/head 1', {
'train loss': train_loss_head1,
'validation loss': val_loss_head1
}, epoch + 1)
writer.add_scalars('loss/head 2', {
'train loss': train_loss_head2,
'validation loss': val_loss_head2
}, epoch + 1)
writer.add_scalars('accuracy/head 0', {
'train accuracy': train_acc,
'validation accuracy': prec1
}, epoch + 1)
writer.add_scalars('accuracy/head 1', {
'train accuracy': train_acc_head1,
'validation accuracy': prec1_head1
}, epoch + 1)
writer.add_scalars('accuracy/head 2', {
'train accuracy': train_acc_head2,
'validation accuracy': prec1_head2
}, epoch + 1)
#for name, param in model.named_parameters():
# writer.add_histogram(name, param.clone().cpu().data.numpy(), epoch + 1)
if args.mimic:
writer.add_scalars('loss/teacher1 student0', {
'train loss': train_loss_t1_s0,
'validation loss': val_loss_t1_s0
}, epoch + 1)
writer.add_scalars('loss/teacher2 student0', {
'train loss': train_loss_t2_s0,
'validation loss': val_loss_t2_s0
}, epoch + 1)
writer.add_scalars('loss/teacher0 student1', {
'train loss': train_loss_t0_s1,
'validation loss': val_loss_t0_s1
}, epoch + 1)
writer.add_scalars('loss/teacher2 student1', {
'train loss': train_loss_t2_s1,
'validation loss': val_loss_t2_s1
}, epoch + 1)
writer.add_scalars('loss/teacher0 student2', {
'train loss': train_loss_t0_s2,
'validation loss': val_loss_t0_s2
}, epoch + 1)
writer.add_scalars('loss/teacher1 student2', {
'train loss': train_loss_t1_s2,
'validation loss': val_loss_t1_s2
}, epoch + 1)
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
writer.close()
print('Best accuracy:')
print(best_prec1)
def main():
global args, best_prec1, cp_logger, lr_schedul
args = parser.parse_args()
if args.seed is not None:
random.seed(args.seed)
#np.random.seed(args.seed)
torch.manual_seed(args.seed)
cudnn.deterministic = True
warnings.warn('You have chosen to seed training. '
'This will turn on the CUDNN deterministic setting, '
'which can slow down your training considerably! '
'You may see unexpected behavior when restarting '
'from checkpoints.')
args.distributed = args.world_size > 1
if args.distributed:
dist.init_process_group(backend=args.dist_backend,
init_method=args.dist_url,
world_size=args.world_size)
print("#Weights to track '{}'".format(args.track_weights))
# create model
print("=> creating model '{}'".format(args.arch))
model = None
if args.arch.startswith('mobilenetv2'):
model = models.__dict__[args.arch](width_mult=args.width_mult)
elif args.arch.startswith('efficientnet'):
print('efficientnet')
model = EfficientNet.from_name(args.arch)
else:
model = models.__dict__[args.arch]()
if not args.distributed:
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
else:
model.cuda()
model = torch.nn.parallel.DistributedDataParallel(model)
model_weights_tot = sum([param.nelement() for param in model.parameters()])
print(model)
print(model_weights_tot)
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
if args.use_qe == 'True':
target_quantile = 1 - (args.track_weights / model_weights_tot)
print('target quantile: ', target_quantile)
else:
target_quantile = None
print('Use exact sort without qe')
print("Baseline Optimizer is used!")
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
if args.lr_decay == 'specified':
lr_schedul = lr_scheduled('schedule.csv')
scheduler = ReduceLROnPlateau(optimizer,
mode='min',
factor=0.2,
patience=5,
verbose=False,
threshold=1e-1,
threshold_mode='abs',
cooldown=0,
min_lr=0.002,
eps=1e-8)
# optionally resume from a checkpoint
title = 'ImageNet-' + args.arch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
args.checkpoint = os.path.dirname(args.resume)
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
cp_logger = Logger(os.path.join(args.checkpoint, 'cp_log.txt'),
title=title,
resume=True)
else:
print("=> no checkpoint found at '{}'".format(args.resume))
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title)
cp_logger = Logger(os.path.join(args.checkpoint, 'cp_log.txt'),
title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
cp_logger.set_names([
'itteration in epoch', 'Train Loss', 'Valid Loss',
'Train Acc.', 'Train Acc. top5', 'Valid Acc.',
'Valid Acc. top5', 'batch speed'
])
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
cp_logger = Logger(os.path.join(args.checkpoint, 'cp_log.txt'),
title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
cp_logger.set_names([
'itteration in epoch', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Train Acc. top5', 'Valid Acc.', 'Valid Acc. top5', 'batch speed'
])
cudnn.benchmark = True
# Data loading code
if args.data_backend == 'pytorch':
get_train_loader = get_pytorch_train_loader
get_val_loader = get_pytorch_val_loader
elif args.data_backend == 'dali-gpu':
get_train_loader = get_dali_train_loader(dali_cpu=False)
get_val_loader = get_dali_val_loader()
elif args.data_backend == 'dali-cpu':
get_train_loader = get_dali_train_loader(dali_cpu=True)
get_val_loader = get_dali_val_loader()
train_loader, train_loader_len = get_train_loader(
args.data,
args.batch_size,
workers=args.workers,
input_size=args.input_size)
val_loader, val_loader_len = get_val_loader(args.data,
args.batch_size,
workers=args.workers,
input_size=args.input_size)
if args.evaluate:
from collections import OrderedDict
if os.path.isfile(args.weight):
print("=> loading pretrained weight '{}'".format(args.weight))
source_state = torch.load(args.weight)
target_state = OrderedDict()
for k, v in source_state.items():
if k[:7] != 'module.':
k = 'module.' + k
target_state[k] = v
model.load_state_dict(target_state)
else:
print("=> no weight found at '{}'".format(args.weight))
validate(val_loader, val_loader_len, model, criterion)
return
# visualization
writer = SummaryWriter(os.path.join(args.checkpoint, 'logs'))
print("Training started!", flush=True)
for epoch in range(args.start_epoch, args.epochs):
if args.distributed:
train_sampler.set_epoch(epoch)
# the folder name to dump logs and files
folder = 'baseline'
best_reported_acc = best_accuracy(current_epoch=epoch,
run_name='ImageNet_' + args.arch,
folder=folder)
print("Best reported Valid Acc. for epoch: " + str(epoch) + " is: " +
str(best_reported_acc),
flush=True)
print('\nEpoch: [%d | %d]' % (epoch + 1, args.epochs), flush=True)
# train for one epoch
train_loss, train_acc = train(train_loader, train_loader_len, model,
criterion, optimizer, epoch)
# evaluate on validation set
val_loss, prec1 = validate(val_loader, val_loader_len, model,
criterion)
lr = optimizer.param_groups[0]['lr']
# append logger file
logger.append([lr, train_loss, val_loss, train_acc, prec1])
# tensorboardX
writer.add_scalar('learning rate', lr, epoch + 1)
writer.add_scalars('loss', {
'train loss': train_loss,
'validation loss': val_loss
}, epoch + 1)
writer.add_scalars('accuracy', {
'train accuracy': train_acc,
'validation accuracy': prec1
}, epoch + 1)
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
if (epoch > 5) & (epoch % 5 == 4): # save every 5 epoch!
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
},
False,
checkpoint=args.checkpoint,
filename=str(epoch + 1) + '_' + 'checkpoint.pth.tar')
if args.lr_decay == 'plateau2':
scheduler.step(val_loss)
# Early terminations
print("Accuracy here: " + str(prec1) + " vs. " +
str(best_reported_acc) + " best accuracy reported!",
flush=True)
if early_terminate(epoch, prec1, best_reported_acc):
print("Early Termination on epoch " + str(epoch))
break
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
cp_logger.close()
#cp_logger.plot()
#savefig(os.path.join(args.checkpoint, 'cp_log.eps'))
writer.close()
print('Best accuracy:')
print(best_prec1, flush=True)
def main():
# global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data
print('==> Preparing dataset %s' % args.dataset)
# train and val
train_transformation = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
eval_transformation = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
data_dir = 'data-local/images/cifar/cifar10/by-image'
num_classes = 10
if args.dataset == 'cifar10':
dataloader = datasets.CIFAR10
else:
dataloader = datasets.CIFAR100
num_classes = 100
data_dir = 'data-local/images/cifar/cifar100/by-image'
dataset_config = {
'train_transformation': train_transformation,
'eval_transformation': eval_transformation,
'datadir': data_dir,
}
trainloader, evalloader, unlabeled = create_data_loaders(**dataset_config,
args=args)
testset = dataloader(root='data-local/workdir/',
train=False,
download=False,
transform=transform_test)
testloader = data.DataLoader(testset,
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers)
# Model
print("==> creating model '{}'".format(args.arch))
if args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
cardinality=args.cardinality,
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.startswith('densenet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
growthRate=args.growthRate,
compressionRate=args.compressionRate,
dropRate=args.drop,
)
elif args.arch.startswith('wrn'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.endswith('resnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
block_name=args.block_name,
)
else:
model = models.__dict__[args.arch](num_classes=num_classes)
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
print(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# Resume
if args.dataset == 'cifar10':
title = 'cifar-10-' + args.arch
else:
title = 'cifar-100-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
# best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate',
'Train Loss',
'Train Acc.',
])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch,
use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
results = []
if args.rand:
print('select samples randomly...')
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(trainloader, model, criterion, optimizer,
epoch, use_cuda)
# test_loss, test_acc = test(testloader, model, criterion, epoch, use_cuda)
# if not args.rand and epoch >= args.epochs - args.epoch_num: # last 5, interval = 1 epoch
interval = args.interval # interval !=1 epochs
if not args.rand and epoch in np.arange(
args.epochs)[-1:-args.epoch_num * interval - 1:-interval]:
results.append(get_p(evalloader, model, epoch, use_cuda))
# append logger file
# logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc])
logger.append([state['lr'], train_loss, train_acc])
# save model
# is_best = test_acc > best_acc
# best_acc = max(test_acc, best_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
# 'acc': test_acc,
# 'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
select_data(args.labels, results, unlabeled, args.select_num)
def main():
global best_acc
# Data
print('==> Preparing data..')
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
])
# data_file = './data'
# data_file = 'C:/Users/dongxian/data/CIFAR'
data_file = '/data/wudongxian/dataset/CIFAR/'
if args.dataset == 'cifar10':
data_name = 'CIFAR10'
num_classes = 10
else:
data_name = 'CIFAR100'
num_classes = 100
trainset = getattr(torchvision.datasets, data_name)(root=data_file, train=True,
download=True, transform=transform_train)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers)
testset = getattr(torchvision.datasets, data_name)(root=data_file, train=False, download=True, transform=transform_test)
testloader = torch.utils.data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers)
# Model
print('==> Building model..')
net = models.__dict__[args.arch](num_classes=num_classes)
teacher_net = models.__dict__[args.arch](num_classes=num_classes)
title = 'noisy_cifar-' + args.arch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Adv Train Loss', 'Nat Train Loss', 'Nat Valid Loss',
'Adv Train Acc', 'Nat Train Loss', 'Nat Valid Acc'])
if torch.cuda.is_available():
net = net.to(device)
net = torch.nn.DataParallel(net)
teacher_net = teacher_net.to(device)
teacher_net = torch.nn.DataParallel(teacher_net)
hard_update(teacher_net, net)
print('Using', torch.cuda.device_count(), 'GPUs.')
cudnn.benchmark = True
print('Using CUDA..')
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay)
schedule_lr = optim.lr_scheduler.MultiStepLR(optimizer, args.schedule, gamma=args.gamma)
adversary = PGDAttack(model=net, epsilon=0.031, num_steps=10, step_size=0.007)
for epoch in range(args.start_epoch, args.epochs):
schedule_lr.step(epoch)
state['lr'] = optimizer.state_dict()['param_groups'][0]['lr']
if args.mode == 'nat':
adv_train_loss, adv_train_acc = train(trainloader, net, teacher_net, criterion, optimizer, epoch)
else:
adv_train_loss, adv_train_acc = train_adv(trainloader, net, teacher_net, criterion, optimizer, epoch, adversary=adversary)
nat_train_loss, nat_train_acc = test(trainloader, net, criterion, epoch)
nat_test_loss, nat_test_acc = test(testloader, net, criterion, epoch)
logger.append([state['lr'], adv_train_loss, nat_train_loss, nat_test_loss,
adv_train_acc, nat_train_acc, nat_test_acc])
# what is the best?
is_best = adv_train_acc > best_acc
best_acc = max(adv_train_acc, best_acc)
save_checkpoint({
'epoch': epoch + 1,
'state_dict': net.state_dict(),
'acc': adv_train_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
}, is_best, checkpoint=args.checkpoint)
if (epoch + 1) % 10 == 0 and epoch > 1:
save_checkpoint({
'epoch': epoch + 1,
'state_dict': net.state_dict(),
'acc': adv_train_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
}, is_best, checkpoint=args.checkpoint, filename='checkpoint_{:0>3d}.pth.tar'.format(epoch+1))
save_checkpoint({
'epoch': epoch + 1,
'state_dict': teacher_net.state_dict(),
'acc': adv_train_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
}, is_best, checkpoint=args.checkpoint, filename='checkpoint_teacher_{:0>3d}.pth.tar'.format(epoch + 1))
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
def main():
global args, best_prec1
args = parser.parse_args()
args.distributed = args.world_size > 1
if args.distributed:
dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
world_size=args.world_size)
# Use CUDA
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
use_cuda = torch.cuda.is_available()
# Random seed
if args.manual_seed is None:
args.manual_seed = random.randint(1, 10000)
random.seed(args.manual_seed)
torch.manual_seed(args.manual_seed)
if use_cuda:
torch.cuda.manual_seed_all(args.manual_seed)
# create model
if args.pretrained:
print("=> using pre-trained model '{}'".format(args.arch))
model = models.__dict__[args.arch](pretrained=True)
elif args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
baseWidth=args.base_width,
cardinality=args.cardinality,
)
elif args.arch.startswith('shufflenet'):
model = models.__dict__[args.arch](
groups=args.groups
)
else:
print("=> creating model '{}'".format(args.arch))
model = models.__dict__[args.arch](input_size=args.input_size,
width_mult=args.width_mult)
if not args.distributed:
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
else:
model.cuda()
model = torch.nn.parallel.DistributedDataParallel(model)
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(), args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# optionally resume from a checkpoint
title = 'ImageNet-' + args.arch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})"
.format(args.resume, checkpoint['epoch']))
args.checkpoint = os.path.dirname(args.resume)
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True)
else:
print("=> no checkpoint found at '{}'".format(args.resume))
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.'])
if args.weightfile:
from collections import OrderedDict
if os.path.isfile(args.weightfile):
print("=> loading weightfile '{}'".format(args.weightfile))
preloaded_params = torch.load(args.weightfile)
new_state_dict = OrderedDict()
for k, v in preloaded_params.items():
if k[:7] != 'module.':
k = 'module.' + k
new_state_dict[k] = v
model.load_state_dict(new_state_dict)
cudnn.benchmark = True
# Data loading code
traindir = os.path.join(args.data, 'train')
valdir = os.path.join(args.data, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_dataset = datasets.ImageFolder(
traindir,
transforms.Compose([
transforms.RandomResizedCrop(args.input_size),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
if args.distributed:
train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset)
else:
train_sampler = None
train_loader = torch.utils.data.DataLoader(
train_dataset, batch_size=args.train_batch, shuffle=(train_sampler is None),
num_workers=args.workers, pin_memory=True, sampler=train_sampler)
val_loader = torch.utils.data.DataLoader(
datasets.ImageFolder(valdir, transforms.Compose([
transforms.Resize(int(256.0*(args.input_size/224.0))),
transforms.CenterCrop(args.input_size),
transforms.ToTensor(),
normalize,
])),
batch_size=args.test_batch, shuffle=False,
num_workers=args.workers, pin_memory=True)
if args.evaluate:
validate(val_loader, model, criterion)
return
# visualization
writer = SummaryWriter(os.path.join(args.checkpoint, 'logs'))
for epoch in range(args.start_epoch, args.epochs):
if args.distributed:
train_sampler.set_epoch(epoch)
lr = adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, lr))
# train for one epoch
train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch)
# evaluate on validation set
val_loss, prec1 = validate(val_loader, model, criterion)
# append logger file
logger.append([lr, train_loss, val_loss, train_acc, prec1])
# tensorboardX
writer.add_scalar('learning rate', lr, epoch + 1)
writer.add_scalars('loss', {'train loss': train_loss, 'validation loss': val_loss}, epoch + 1)
writer.add_scalars('accuracy', {'train accuracy': train_acc, 'validation accuracy': prec1}, epoch + 1)
for name, param in model.named_parameters():
writer.add_histogram(name, param.clone().cpu().data.numpy(), epoch + 1)
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
save_checkpoint({
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer' : optimizer.state_dict(),
}, is_best, checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best accuracy:')
print(best_prec1)
# standardize columns
X = utils.standardize_cols(X)
model = PCA(k=2)
model.fit(X)
Z = model.compress(X)
fig, ax = plt.subplots()
ax.scatter(Z[:,0], Z[:,1])
plt.ylabel('z2')
plt.xlabel('z1')
plt.title('PCA')
for i in range(n):
ax.annotate(animals[i], (Z[i,0], Z[i,1]))
utils.savefig('q3_2_PCA_animals.png')
# code below isn't required.
variance_explained = 1 - norm(model.expand(Z) - X, 'fro')**2 / norm(X, 'fro')**2
print('Variance explained {}'.format(variance_explained))
if question == '4.1':
X = utils.load_dataset('highway')['X'].astype(float)/255
n,d = X.shape
h,w = 64,64 # height and width of each image
# the two variables below are parameters for the foreground/background extraction method
# you should just leave these two as default.
k = 5 # number of PCs
threshold = 0.04 # a threshold for separating foreground from background
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data
transform = get_transforms(input_size=args.image_size,
test_size=args.image_size,
backbone=None)
print('==> Preparing dataset %s' % args.trainroot)
trainset = dataset.Dataset(root=args.trainroot,
transform=transform['train'])
train_loader = data.DataLoader(trainset,
batch_size=args.train_batch,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
valset = dataset.TestDataset(root=args.valroot, transform=transform['val'])
val_loader = data.DataLoader(valset,
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
model = make_model(args)
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
print(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = get_optimizer(model, args)
# 基于标准的学习率更新
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode='min',
factor=0.2,
patience=5,
verbose=False)
# Resume
title = 'ImageNet-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.module.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
'Valid Acc.'
])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(val_loader, model, criterion, start_epoch,
use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
print('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, optimizer.param_groups[0]['lr']))
train_loss, train_acc, train_5 = train(train_loader, model, criterion,
optimizer, epoch, use_cuda)
test_loss, test_acc, test_5 = test(val_loader, model, criterion, epoch,
use_cuda)
scheduler.step(test_loss)
# append logger file
logger.append(
[state['lr'], train_loss, test_loss, train_acc, test_acc])
print(
'train_loss:%f, val_loss:%f, train_acc:%f, train_5:%f, val_acc:%f, val_5:%f'
% (train_loss, test_loss, train_acc, train_5, test_acc, test_5))
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
if len(args.gpu_id) > 1:
save_checkpoint(
{
'fold': 0,
'epoch': epoch + 1,
'state_dict': model.module.state_dict(),
'train_acc': train_acc,
'acc': test_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
single=True,
checkpoint=args.checkpoint)
else:
save_checkpoint(
{
'fold': 0,
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'train_acc': train_acc,
'acc': test_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
single=True,
checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
d["system_pos"] = d.apply(
lambda r: system_pos[(r["num_particles"], r["freq"])], axis=1)
width = 3.2
height = 3.0
fig, ax = plt.subplots(figsize=(width, height))
for inter, g in sorted(d.groupby("interaction"),
key=lambda x: interaction_pos[x[0]]):
g.sort_values(["freq", "num_particles"], inplace=True,
ascending=[False, True])
ax.plot(g["system_pos"],
abs(g["slope"]),
linestyle="",
linewidth=2,
marker=interaction_markers[inter],
markerfacecolor=interaction_markerfacecolors[inter],
color=interaction_colors[inter],
label=interaction_label[inter])
ax.set_xticks(range(len(systems)))
ax.set_xticklabels(systems, rotation=90)
ax.set_xlabel(r"$(N, \omega)$")
ax.set_ylabel(r"$|\rho_{15}|$")
ax.set_yscale("log")
ax.legend(loc="center left", frameon=False,
title="$(\sigma_{\mathrm{A}}, \sigma_{\mathrm{B}})$",
bbox_to_anchor=(1.0 - 0.3 / width, 0.5))
fig.tight_layout()
fig.subplots_adjust(left=0.2, right=0.75)
utils.savefig(fig, fn="../Manuscript/fig-rel-slopes2.pdf")
(d["num_shells"] == num_shells) &
(d["num_filled"] == num_filled) &
# filter out higher frequencies because they stretch the plot too much
(d["freq"] <= 1.0) &
(d["ml"] == ml)]
num_particles = num_filled * (num_filled + 1)
energy_type = {"ground": "ground state",
"add": "addition", "rm": "removal"}[label]
fig, ax = plt.subplots()
fig.set_size_inches((4, 3))
for method, case in d.groupby("method"):
case = case.sort_values("freq").drop_duplicates()
xs = case["freq"]
ys_ref = xs.map(lambda freq: d[(d["method"] == "hf") &
(d["freq"] == freq)]["energy"])
ys = case["energy"] / ys_ref
ax.plot(xs, ys, "-x", label=utils.METHOD_LABEL[method],
color=utils.METHOD_COLOR[method])
if kwargs["title"]:
ax.set_title("{energy_type} energy for {num_particles} particles "
"(ML = {ml}, K = {num_shells})"
.format(**locals()))
ax.set_xlabel("ω")
ax.set_ylabel("ε/ε[HF]")
ax.legend()
fig.tight_layout()
utils.savefig(fig,
"by-freq-{num_shells}-{num_particles}-"
"{label}-{ml}-{interaction}"
.format(**locals()))
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data
print('==> Preparing dataset %s' % args.dataset)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
if args.dataset == 'cifar10':
dataloader = datasets.CIFAR10
num_classes = 10
else:
dataloader = datasets.CIFAR100
num_classes = 100
trainset = dataloader(root='./data',
train=True,
download=True,
transform=transform_train)
trainloader = data.DataLoader(trainset,
batch_size=args.train_batch,
shuffle=True,
num_workers=args.workers)
testset = dataloader(root='./data',
train=False,
download=False,
transform=transform_test)
testloader = data.DataLoader(testset,
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers)
# Model
print("==> creating model '{}'".format(args.arch))
if args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
cardinality=args.cardinality,
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.startswith('densenet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
growthRate=args.growthRate,
compressionRate=args.compressionRate,
dropRate=args.drop,
)
elif args.arch.startswith('wrn'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
widen_factor=args.widen_factor,
dropRate=args.drop,
)
elif args.arch.startswith('resnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
block_name=args.block_name,
)
elif args.arch.startswith('preresnet'):
model = models.__dict__[args.arch](
num_classes=num_classes,
depth=args.depth,
block_name=args.block_name,
)
elif args.arch.startswith('horesnet'):
model = models.__dict__[args.arch](num_classes=num_classes,
depth=args.depth,
eta=args.eta,
block_name=args.block_name,
feature_vec=args.feature_vec)
elif args.arch.startswith('hopreresnet'):
model = models.__dict__[args.arch](num_classes=num_classes,
depth=args.depth,
eta=args.eta,
block_name=args.block_name,
feature_vec=args.feature_vec)
elif args.arch.startswith('nagpreresnet'):
model = models.__dict__[args.arch](num_classes=num_classes,
depth=args.depth,
eta=args.eta,
block_name=args.block_name,
feature_vec=args.feature_vec)
elif args.arch.startswith('mompreresnet'):
model = models.__dict__[args.arch](num_classes=num_classes,
depth=args.depth,
eta=args.eta,
block_name=args.block_name,
feature_vec=args.feature_vec)
elif args.arch.startswith('v2_preresnet'):
model = models.__dict__[args.arch](block_name='basicblock',
num_blocks=[2, 2, 2, 2],
num_classes=num_classes)
else:
print('Model is specified wrongly - Use standard model')
model = models.__dict__[args.arch](num_classes=num_classes)
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
print(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
criterion = nn.CrossEntropyLoss()
if args.optimizer.lower() == 'sgd':
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# elif args.optimizer.lower() == 'adam':
# optimizer = optim.Adam(model.parameters(), lr=args.lr, betas=(args.beta1, args.beta2), weight_decay=args.weight_decay)
elif args.optimizer.lower() == 'radam':
optimizer = RAdam(model.parameters(),
lr=args.lr,
betas=(args.beta1, args.beta2),
weight_decay=args.weight_decay)
elif args.optimizer.lower() == 'adamw':
optimizer = AdamW(model.parameters(),
lr=args.lr,
betas=(args.beta1, args.beta2),
weight_decay=args.weight_decay,
warmup=args.warmup)
elif args.optimizer.lower() == 'adam':
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
betas=(args.beta1, args.beta2),
weight_decay=args.weight_decay)
elif args.optimizer.lower() == 'srsgd':
iter_count = 1
optimizer = SGD_Adaptive(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay,
iter_count=iter_count,
restarting_iter=40)
elif args.optimizer.lower() == 'sradam':
iter_count = 1
optimizer = SRNAdam(model.parameters(),
lr=args.lr,
iter_count=iter_count,
restarting_iter=100)
# Resume
title = 'cifar-10-' + args.arch
# if args.resume:
# # Load checkpoint.
# print('==> Resuming from checkpoint..')
# assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!'
# args.checkpoint = os.path.dirname(args.resume)
# checkpoint = torch.load(args.resume)
# best_acc = checkpoint['best_acc']
# start_epoch = checkpoint['epoch']
# model.load_state_dict(checkpoint['state_dict'])
# optimizer.load_state_dict(checkpoint['optimizer'])
# logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True)
# else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.'
])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch,
use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
if args.optimizer.lower() == 'srsgd':
if epoch == args.schedule[0]:
optimizer = SGD_Adaptive(model.parameters(),
lr=args.lr * args.gamma,
weight_decay=args.weight_decay,
iter_count=iter_count,
restarting_iter=80)
elif epoch == args.schedule[1]:
optimizer = SGD_Adaptive(model.parameters(),
lr=args.lr * (args.gamma**2),
weight_decay=args.weight_decay,
iter_count=iter_count,
restarting_iter=200)
elif epoch == args.schedule[2]:
optimizer = SGD_Adaptive(model.parameters(),
lr=args.lr * (args.gamma**3),
weight_decay=args.weight_decay,
iter_count=iter_count,
restarting_iter=500)
elif epoch == args.schedule[3]:
optimizer = SGD_Adaptive(model.parameters(),
lr=args.lr * (args.gamma**4),
weight_decay=args.weight_decay,
iter_count=iter_count,
restarting_iter=1000)
elif args.optimizer.lower() == 'sradam':
if epoch == 80:
optimizer = SRNAdam(model.parameters(),
lr=args.lr * args.gamma,
iter_count=iter_count,
restarting_iter=500) #500 done
elif epoch == 160:
optimizer = SRNAdam(model.parameters(),
lr=args.lr * (args.gamma**2),
iter_count=iter_count,
restarting_iter=1000) #1000 done
elif args.arch.startswith('wrn'):
if args.optimizer.lower() == 'sgd':
if epoch == 60:
optimizer = optim.SGD(model.parameters(),
lr=0.02,
momentum=args.momentum,
weight_decay=args.weight_decay)
elif epoch == 120:
optimizer = optim.SGD(model.parameters(),
lr=0.004,
momentum=args.momentum,
weight_decay=args.weight_decay)
elif epoch == 160:
optimizer = optim.SGD(model.parameters(),
lr=0.0008,
momentum=args.momentum,
weight_decay=args.weight_decay)
else:
adjust_learning_rate(optimizer, epoch)
logger.file.write('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
if args.optimizer.lower() == 'srsgd' or args.optimizer.lower(
) == 'sradam':
train_loss, train_acc, iter_count = train(trainloader, model,
criterion, optimizer,
epoch, use_cuda, logger)
else:
train_loss, train_acc = train(trainloader, model, criterion,
optimizer, epoch, use_cuda, logger)
test_loss, test_acc = test(testloader, model, criterion, epoch,
use_cuda, logger)
# append logger file
logger.append(
[state['lr'], train_loss, test_loss, train_acc, test_acc])
writer.add_scalars('train_loss', {args.model_name: train_loss}, epoch)
writer.add_scalars('test_loss', {args.model_name: test_loss}, epoch)
writer.add_scalars('train_acc', {args.model_name: train_acc}, epoch)
writer.add_scalars('test_acc', {args.model_name: test_acc}, epoch)
# writer.add_scalars('loss_tracking/train_loss', {args.model_name: train_loss}, epoch)
# writer.add_scalars('loss_tracking/test_loss', {args.model_name: test_loss}, epoch)
# writer.add_scalars('loss_tracking/train_acc', {args.model_name: train_acc}, epoch)
# writer.add_scalars('loss_tracking/test_acc', {args.model_name: test_acc}, epoch)
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
logger.file.write('Best acc:%f' % best_acc)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data loading code
traindir = os.path.join(args.data, 'train')
valdir = os.path.join(args.data, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_loader = torch.utils.data.DataLoader(
datasets.ImageFolder(traindir, transforms.Compose([
transforms.RandomSizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
])),
batch_size=args.train_batch, shuffle=True,
num_workers=args.workers, pin_memory=True)
val_loader = torch.utils.data.DataLoader(
datasets.ImageFolder(valdir, transforms.Compose([
transforms.Scale(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size=args.test_batch, shuffle=False,
num_workers=args.workers, pin_memory=True)
# create model
if args.pretrained:
print("=> using pre-trained model '{}'".format(args.arch))
model = models.__dict__[args.arch](pretrained=True)
elif args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
baseWidth=args.base_width,
cardinality=args.cardinality,
)
else:
print("=> creating model '{}'".format(args.arch))
model = models.__dict__[args.arch]()
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters())/1000000.0))
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay)
# Resume
title = 'ImageNet-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.'])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(val_loader, model, criterion, start_epoch, use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch, use_cuda)
test_loss, test_acc = test(val_loader, model, criterion, epoch, use_cuda)
# append logger file
logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer' : optimizer.state_dict(),
}, is_best, checkpoint=args.checkpoint)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
def main():
global best_top1, best_top5
args.world_size = 1
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
# Data loading code
traindir = os.path.join(args.data, 'train')
valdir = os.path.join(args.data, 'val')
crop_size = 224
val_size = 256
pipe = HybridTrainPipe(batch_size=args.train_batch,
num_threads=args.workers,
device_id=args.local_rank,
data_dir=traindir,
crop=crop_size,
dali_cpu=args.dali_cpu)
pipe.build()
train_loader = DALIClassificationIterator(
pipe, size=int(pipe.epoch_size("Reader") / args.world_size))
pipe = HybridValPipe(batch_size=args.test_batch,
num_threads=args.workers,
device_id=args.local_rank,
data_dir=valdir,
crop=crop_size,
size=val_size)
pipe.build()
val_loader = DALIClassificationIterator(
pipe, size=int(pipe.epoch_size("Reader") / args.world_size))
# create model
if args.pretrained:
print("=> using pre-trained model '{}'".format(args.arch))
model = models.__dict__[args.arch](pretrained=True)
elif args.arch.startswith('resnext'):
model = models.__dict__[args.arch](
baseWidth=args.base_width,
cardinality=args.cardinality,
)
else:
print("=> creating model '{}'".format(args.arch))
model = models.__dict__[args.arch]()
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
if args.optimizer.lower() == 'sgd':
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
elif args.optimizer.lower() == 'adam':
optimizer = AdamW(model.parameters(),
lr=args.lr,
betas=(args.beta1, args.beta2),
weight_decay=args.weight_decay)
elif args.optimizer.lower() == 'radam':
optimizer = RAdam(model.parameters(),
lr=args.lr,
betas=(args.beta1, args.beta2),
weight_decay=args.weight_decay)
elif args.optimizer.lower() == 'lsadam':
optimizer = LSAdamW(model.parameters(),
lr=args.lr * ((1. + 4. * args.sigma)**(0.25)),
betas=(args.beta1, args.beta2),
weight_decay=args.weight_decay,
sigma=args.sigma)
elif args.optimizer.lower() == 'lsradam':
sigma = 0.1
optimizer = LSRAdam(model.parameters(),
lr=args.lr * ((1. + 4. * args.sigma)**(0.25)),
betas=(args.beta1, args.beta2),
weight_decay=args.weight_decay,
sigma=args.sigma)
# Resume
title = 'ImageNet-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isfile(
args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = os.path.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_top1 = checkpoint['best_top1']
best_top5 = checkpoint['best_top5']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
title=title,
resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names([
'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Top1',
'Valid Top1', 'Train Top5', 'Valid Top5'
])
logger.file.write(' Total params: %.2fM' %
(sum(p.numel() for p in model.parameters()) / 1000000.0))
if args.evaluate:
logger.file.write('\nEvaluation only')
test_loss, test_top1, test_top5 = test(val_loader, model, criterion,
start_epoch, use_cuda, logger)
logger.file.write(
' Test Loss: %.8f, Test Top1: %.2f, Test Top5: %.2f' %
(test_loss, test_top1, test_top5))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
logger.file.write('\nEpoch: [%d | %d] LR: %f' %
(epoch + 1, args.epochs, state['lr']))
train_loss, train_top1, train_top5 = train(train_loader, model,
criterion, optimizer, epoch,
use_cuda, logger)
test_loss, test_top1, test_top5 = test(val_loader, model, criterion,
epoch, use_cuda, logger)
# append logger file
logger.append([
state['lr'], train_loss, test_loss, train_top1, test_top1,
train_top5, test_top5
])
writer.add_scalars('train_loss', {args.model_name: train_loss}, epoch)
writer.add_scalars('test_loss', {args.model_name: test_loss}, epoch)
writer.add_scalars('train_top1', {args.model_name: train_top1}, epoch)
writer.add_scalars('test_top1', {args.model_name: test_top1}, epoch)
writer.add_scalars('train_top5', {args.model_name: train_top5}, epoch)
writer.add_scalars('test_top5', {args.model_name: test_top5}, epoch)
# save model
is_best = test_top1 > best_top1
best_top1 = max(test_top1, best_top1)
best_top5 = max(test_top5, best_top5)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'top1': test_top1,
'top5': test_top5,
'best_top1': best_top1,
'best_top5': best_top5,
'optimizer': optimizer.state_dict(),
},
is_best,
checkpoint=args.checkpoint)
# reset DALI iterators
train_loader.reset()
val_loader.reset()
logger.file.write('Best top1:')
logger.file.write(best_top1)
logger.file.write('Best top5:')
logger.file.write(best_top5)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best top1:')
print(best_top1)
print('Best top5:')
print(best_top5)
