def test_accuray_ens(model, ckpt_list, loaders, name):
acc_dir = dict()
acc_list = []
predictions_sum = np.zeros((len(loaders['test'].dataset), num_classes))
for i, path in enumerate(ckpt_list):
print(path)
temp = np.zeros((2))
checkpoint = torch.load(path)
model.load_state_dict(checkpoint['model_state'])
predictions, targets = utils.predictions(loaders['test'], model)
acc = 100.0 * np.mean(np.argmax(predictions, axis=1) == targets)
predictions_sum += predictions
ens_acc = 100.0 * np.mean(
np.argmax(predictions_sum, axis=1) == targets)
temp[0] = acc
temp[1] = ens_acc
acc_list.append(temp)
acc_dir[i] = temp
print('Model accuracy: %8.4f. Ensemble accuracy: %8.4f' %
(acc, ens_acc))
acc_value = np.vstack(acc_list)
np.savez(os.path.join(args.dir,
'eval_ensemble' + name + args.model + '.npz'),
ens_value=acc_value)
return acc_dir
def test_linear_regression(self):
initial_theta = np.ones(self.n + 1)
lamda = 0.0
theta = train_linear_reg(initial_theta, self.X, self.y, lamda)
hypo = predictions(theta, self.X)
# now scatter the data and plot the hypothesis
plt.xlabel("Change in water level (x)")
plt.ylabel("Water flowing out of the dam (y)")
plt.scatter(self.X, self.y, marker='x', c='r', s=30, linewidth=2)
plt.plot(self.X, hypo)
plt.show()
def test_linear_regression(self):
initial_theta = np.ones(self.n+1)
lamda = 0.0
theta = train_linear_reg(initial_theta, self.X, self.y, lamda)
hypo = predictions(theta, self.X)
# now scatter the data and plot the hypothesis
plt.xlabel("Change in water level (x)")
plt.ylabel("Water flowing out of the dam (y)")
plt.scatter( self.X, self.y, marker='x', c='r', s=30, linewidth=2 )
plt.plot( self.X, hypo )
plt.show()
def test_linear_regression_poly(self):
highest_degree = 8
X_poly = map_poly_features(self.X, highest_degree)
X_poly, mu, sigma = feature_normalization(X_poly)
initial_theta = np.ones(highest_degree+1)
lamda = 0.0
theta = train_linear_reg(initial_theta, X_poly, self.y, lamda)
hypo = predictions(theta, X_poly)
# now scatter the data and plot the hypothesis
df = pd.DataFrame(np.hstack(( self.X, hypo.reshape(self.y.shape), self.y )), columns=['X','hypo','y'])
df = df.sort('X')
plt.xlabel("Change in water level (x)")
plt.ylabel("Water flowing out of the dam (y)")
plt.scatter( df['X'], df['y'], marker='x', c='r', s=30, linewidth=2 )
plt.plot( df['X'], df['hypo'], linestyle='--', linewidth=3 )
plt.show()
def test_linear_regression_poly(self):
highest_degree = 8
X_poly = map_poly_features(self.X, highest_degree)
X_poly, mu, sigma = feature_normalization(X_poly)
initial_theta = np.ones(highest_degree + 1)
lamda = 0.0
theta = train_linear_reg(initial_theta, X_poly, self.y, lamda)
hypo = predictions(theta, X_poly)
# now scatter the data and plot the hypothesis
df = pd.DataFrame(np.hstack(
(self.X, hypo.reshape(self.y.shape), self.y)),
columns=['X', 'hypo', 'y'])
df = df.sort('X')
plt.xlabel("Change in water level (x)")
plt.ylabel("Water flowing out of the dam (y)")
plt.scatter(df['X'], df['y'], marker='x', c='r', s=30, linewidth=2)
plt.plot(df['X'], df['hypo'], linestyle='--', linewidth=3)
plt.show()
def test_accuray_ens_vote(model, ckpt_list, loaders):
predictions_list = []
for i, path in enumerate(ckpt_list):
checkpoint = torch.load(path)
model.load_state_dict(checkpoint['model_state'])
prediction_temp, targets = utils.predictions(loaders['test'], model)
prediction1 = np.argmax(prediction_temp, axis=1)
predictions_list.append(prediction1)
prediction_finally = np.zeros((prediction_temp.shape))
for i, item in enumerate(predictions_list):
for k in range(prediction_finally.shape[0]):
pred_value = item[k]
prediction_finally[k][
pred_value] = prediction_finally[k][pred_value] + 1
ens_acc = 100.0 * np.mean(np.argmax(prediction_finally, axis=1) == targets)
print('Ensemble accuracy: %8.4f' % (ens_acc))
def main():
parser = argparse.ArgumentParser(description='DNN curve training')
parser.add_argument('--dir',
type=str,
default='/tmp/curve/',
metavar='DIR',
help='training directory (default: /tmp/curve/)')
parser.add_argument('--dataset',
type=str,
default='CIFAR10',
metavar='DATASET',
help='dataset name (default: CIFAR10)')
parser.add_argument(
'--use_test',
action='store_true',
help='switches between validation and test set (default: validation)')
parser.add_argument('--transform',
type=str,
default='VGG',
metavar='TRANSFORM',
help='transform name (default: VGG)')
parser.add_argument('--data_path',
type=str,
default=None,
metavar='PATH',
help='path to datasets location (default: None)')
parser.add_argument('--batch_size',
type=int,
default=128,
metavar='N',
help='input batch size (default: 128)')
parser.add_argument('--num-workers',
type=int,
default=4,
metavar='N',
help='number of workers (default: 4)')
parser.add_argument('--model',
type=str,
default=None,
metavar='MODEL',
required=True,
help='model name (default: None)')
parser.add_argument('--comment',
type=str,
default="",
metavar='T',
help='comment to the experiment')
parser.add_argument(
'--resume',
type=str,
default=None,
metavar='CKPT',
help='checkpoint to resume training from (default: None)')
parser.add_argument('--epochs',
type=int,
default=200,
metavar='N',
help='number of epochs to train (default: 200)')
parser.add_argument('--save_freq',
type=int,
default=50,
metavar='N',
help='save frequency (default: 50)')
parser.add_argument('--print_freq',
type=int,
default=1,
metavar='N',
help='print frequency (default: 1)')
parser.add_argument('--lr',
type=float,
default=0.01,
metavar='LR',
help='initial learning rate (default: 0.01)')
parser.add_argument('--momentum',
type=float,
default=0.9,
metavar='M',
help='SGD momentum (default: 0.9)')
parser.add_argument('--wd',
type=float,
default=1e-4,
metavar='WD',
help='weight decay (default: 1e-4)')
parser.add_argument('--seed',
type=int,
default=1,
metavar='S',
help='random seed (default: 1)')
parser.add_argument('--width',
type=int,
default=64,
metavar='N',
help='width of 1 network')
parser.add_argument('--num-nets',
type=int,
default=8,
metavar='N',
help='number of networks in ensemble')
parser.add_argument('--num-exps',
type=int,
default=3,
metavar='N',
help='number of times for executung the whole script')
parser.add_argument('--not-random-dir',
action='store_true',
help='randomize dir')
parser.add_argument('--dropout',
type=float,
default=0.5,
metavar='WD',
help='dropout rate for fully-connected layers')
parser.add_argument('--not-save-weights',
action='store_true',
help='not save weights')
parser.add_argument('--lr-shed',
type=str,
default='standard',
metavar='LRSHED',
help='lr shedule name (default: standard)')
parser.add_argument('--shorten_dataset',
action='store_true',
help='same train set of size N/num_nets for each net')
args = parser.parse_args()
letters = string.ascii_lowercase
exp_label = "%s_%s/%s" % (args.dataset, args.model, args.comment)
if args.num_exps > 1:
if not args.not_random_dir:
exp_label += "_%s/" % ''.join(
random.choice(letters) for i in range(5))
else:
exp_label += "/"
np.random.seed(args.seed)
for exp_num in range(args.num_exps):
args.seed = np.random.randint(1000)
fmt_list = [('lr', "3.4e"), ('tr_loss', "3.3e"), ('tr_acc', '9.4f'), \
('te_nll', "3.3e"), ('te_acc', '9.4f'), ('ens_acc', '9.4f'),
('ens_nll', '3.3e'), ('time', ".3f")]
fmt = dict(fmt_list)
log = logger.Logger(exp_label, fmt=fmt, base=args.dir)
log.print(" ".join(sys.argv))
log.print(args)
torch.backends.cudnn.benchmark = True
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
loaders, num_classes = data.loaders(args.dataset, args.data_path,
args.batch_size, args.num_workers,
args.transform, args.use_test)
if args.shorten_dataset:
loaders["train"].dataset.targets = loaders[
"train"].dataset.targets[:5000]
loaders["train"].dataset.data = loaders[
"train"].dataset.data[:5000]
architecture = getattr(models, args.model)()
architecture.kwargs["k"] = args.width
if "VGG" in args.model or "WideResNet" in args.model:
architecture.kwargs["p"] = args.dropout
if args.lr_shed == "standard":
def learning_rate_schedule(base_lr, epoch, total_epochs):
alpha = epoch / total_epochs
if alpha <= 0.5:
factor = 1.0
elif alpha <= 0.9:
factor = 1.0 - (alpha - 0.5) / 0.4 * 0.99
else:
factor = 0.01
return factor * base_lr
elif args.lr_shed == "stair":
def learning_rate_schedule(base_lr, epoch, total_epochs):
if epoch < total_epochs / 2:
factor = 1.0
else:
factor = 0.1
return factor * base_lr
elif args.lr_shed == "exp":
def learning_rate_schedule(base_lr, epoch, total_epochs):
factor = 0.9885**epoch
return factor * base_lr
criterion = F.cross_entropy
regularizer = None
ensemble_size = 0
predictions_sum = np.zeros((len(loaders['test'].dataset), num_classes))
for num_model in range(args.num_nets):
model = architecture.base(num_classes=num_classes,
**architecture.kwargs)
model.cuda()
optimizer = torch.optim.SGD(filter(
lambda param: param.requires_grad, model.parameters()),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.wd)
start_epoch = 1
if args.resume is not None:
print('Resume training from %s' % args.resume)
checkpoint = torch.load(args.resume)
start_epoch = checkpoint['epoch'] + 1
model.load_state_dict(checkpoint['model_state'])
optimizer.load_state_dict(checkpoint['optimizer_state'])
has_bn = utils.check_bn(model)
test_res = {'loss': None, 'accuracy': None, 'nll': None}
for epoch in range(start_epoch, args.epochs + 1):
time_ep = time.time()
lr = learning_rate_schedule(args.lr, epoch, args.epochs)
utils.adjust_learning_rate(optimizer, lr)
train_res = utils.train(loaders['train'], model, optimizer,
criterion, regularizer)
ens_acc = None
ens_nll = None
if epoch == args.epochs:
predictions_logits, targets = utils.predictions(
loaders['test'], model)
predictions = F.softmax(
torch.from_numpy(predictions_logits), dim=1).numpy()
predictions_sum = ensemble_size/(ensemble_size+1) \
* predictions_sum+\
predictions/(ensemble_size+1)
ensemble_size += 1
ens_acc = 100.0 * np.mean(
np.argmax(predictions_sum, axis=1) == targets)
predictions_sum_log = np.log(predictions_sum + 1e-15)
ens_nll = -metrics.metrics_kfold(predictions_sum_log, targets, n_splits=2, n_runs=5,\
verbose=False, temp_scale=True)["ll"]
np.save(log.path + '/predictions_run%d' % num_model,
predictions_logits)
if not args.not_save_weights and epoch % args.save_freq == 0:
utils.save_checkpoint(
log.get_checkpoint(epoch),
epoch,
model_state=model.state_dict(),
optimizer_state=optimizer.state_dict())
time_ep = time.time() - time_ep
if epoch % args.print_freq == 0:
test_res = utils.test(loaders['test'], model, \
criterion, regularizer)
values = [
lr, train_res['loss'], train_res['accuracy'],
test_res['nll'], test_res['accuracy'], ens_acc,
ens_nll, time_ep
]
for (k, _), v in zip(fmt_list, values):
log.add(epoch, **{k: v})
log.iter_info()
log.save(silent=True)
if not args.not_save_weights:
utils.save_checkpoint(log.path +
'/model_run%d.cpt' % num_model,
args.epochs,
model_state=model.state_dict(),
optimizer_state=optimizer.state_dict())
return log.path
weight_decay=args.wd
)
optimizer.load_state_dict(checkpoint['optimizer_state'])
ensemble_size = 0
predictions_sum = np.zeros((len(loaders['test'].dataset), num_classes))
columns = ['ep', 'lr', 'tr_loss', 'tr_acc', 'te_nll', 'te_acc', 'ens_acc', 'time']
for epoch in range(args.epochs):
time_ep = time.time()
lr_schedule = utils.cyclic_learning_rate(epoch, args.cycle, args.lr_1, args.lr_2)
train_res = utils.train(loaders['train'], model, optimizer, criterion, lr_schedule=lr_schedule)
test_res = utils.test(loaders['test'], model, criterion)
time_ep = time.time() - time_ep
predictions, targets = utils.predictions(loaders['test'], model)
ens_acc = None
if (epoch % args.cycle + 1) == args.cycle // 2:
ensemble_size += 1
predictions_sum += predictions
ens_acc = 100.0 * np.mean(np.argmax(predictions_sum, axis=1) == targets)
if (epoch + 1) % (args.cycle // 2) == 0:
utils.save_checkpoint(
args.dir,
start_epoch + epoch,
name='fge',
model_state=model.state_dict(),
optimizer_state=optimizer.state_dict()
)
columns = [
"ep", "lr", "tr_loss", "tr_acc", "te_nll", "te_acc", "ens_acc", "time"
]
for epoch in range(args.epochs):
time_ep = time.time()
lr_schedule = utils.cyclic_learning_rate(epoch, args.cycle, args.lr_1,
args.lr_2)
train_res = utils.train(loaders["train"],
model,
optimizer,
criterion,
lr_schedule=lr_schedule)
test_res = utils.test(loaders["test"], model, criterion)
time_ep = time.time() - time_ep
predictions, targets = utils.predictions(loaders["test"], model)
ens_acc = None
if (epoch % args.cycle + 1) == args.cycle // 2:
ensemble_size += 1
predictions_sum += predictions
ens_acc = 100.0 * np.mean(
np.argmax(predictions_sum, axis=1) == targets)
if (epoch + 1) % (args.cycle // 2) == 0:
utils.save_checkpoint(args.dir,
start_epoch + epoch,
name="fge",
model_state=model.state_dict(),
optimizer_state=optimizer.state_dict())
values = [
