def test(args, model_load, imageSaver=None, EvaluatorIn=None):
model = get_model(args)
if model_load is not None:
model.load_state_dict(torch.load(model_load), strict=False)
logs = test_with_loaded_model(args, model, imageSaver, EvaluatorIn)
return logs
def __init__(self, hparams: Namespace) -> None:
super(MMClassifier, self).__init__()
self.hparams = hparams
self.model = get_model(self.hparams)
if hparams.task_type == "multilabel":
if hparams.weight_classes:
freqs = [hparams.label_freqs[l] for l in hparams.labels]
label_weights = (torch.FloatTensor(freqs) /
hparams.train_data_len)**-1
self.criterion = nn.BCEWithLogitsLoss(
pos_weight=label_weights.cuda())
else:
self.criterion = nn.BCEWithLogitsLoss()
else:
self.criterion = nn.CrossEntropyLoss()
def train_normal(args, num_epoch, model_save=None, model_load=None, imgSaver=None, EvaluatorIn=None):
# model_save is the name of the file name to save model to
# model_load is the model to initialize training with
if EvaluatorIn is None:
evaluator = Evaluator(args)
else:
evaluator = EvaluatorIn(args)
model = get_model(args)
losses = get_losses(args)
optimizer = get_optimizer(args, model)
scheduler = get_scheduler(args, optimizer)
valid_metric = None
main_metric = evaluator.main_metric
if model_load is not None:
model.load_state_dict(torch.load(model_load), strict=False)
train_dataset, valid_dataset = get_train_val_datasets(args)
valid_logs, train_logs, best_loss_valid_metric = train(model, optimizer, scheduler, losses, train_dataset,
valid_dataset, num_epoch, args, evaluator, imgSaver)
if args['valid_dataset'] and args['train_with_metric']:
valid_metric = valid_logs['metrics'][main_metric]
if args['save_best_loss_name'] is not None or args['save_on_best_total_training_loss']:
valid_metric = best_loss_valid_metric
if not args['valid_dataset']:
valid_metric = train_logs['metrics'][main_metric]
valid_logs = train_logs
if model_save is not None:
torch.save(model.state_dict(), model_save)
return valid_logs, train_logs, valid_metric
def main():
parser = argparse.ArgumentParser(description='Chainer CIFAR example:')
parser.add_argument('--model', default='c3f2')
parser.add_argument('--batchsize', '-b', type=int, default=64)
parser.add_argument('--learnrate', '-l', type=float, default=0.05)
parser.add_argument('--epoch', '-e', type=int, default=300)
parser.add_argument('--gpu', '-g', type=int, default=0)
parser.add_argument('--N', type=int, default=9)
parser.add_argument('--k', type=int, default=10)
parser.add_argument('--out', '-o', default='result')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--resume', '-r', default='')
args = parser.parse_args()
start = time.time()
logger.initialize("outputs_" + args.model)
logger.info(vars(args))
save_dir = logger.get_savedir()
logger.info("Written to {}".format(save_dir))
logger.info('GPU: {}'.format(args.gpu))
logger.info('# Minibatch-size: {}'.format(args.batchsize))
logger.info('# epoch: {}'.format(args.epoch))
logger.info('')
train, test = get_cifar10()
if args.debug:
train, test = train[:200], test[:200]
train_count, test_count = len(train), len(test)
model = get_model(args.model, args.gpu, args.resume)
optimizer = chainer.optimizers.MomentumSGD(args.learnrate)
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.WeightDecay(5e-4))
train_iter = chainer.iterators.SerialIterator(train, args.batchsize)
test_iter = chainer.iterators.SerialIterator(test,
args.batchsize,
repeat=False,
shuffle=False)
sum_accuracy = 0
sum_loss = 0
st = time.time()
iter_cnt = 0
chainer.config.train = True
chainer.config.enable_backprop = True
while train_iter.epoch < args.epoch:
batch = train_iter.next()
if train_iter.epoch % 60 == 0 and train_iter.is_new_epoch:
optimizer.lr *= 0.2
logger.info('Reducing learning rate to: {}'.format(optimizer.lr))
x_array, t_array = convert.concat_examples(batch, args.gpu)
x = chainer.Variable(x_array)
t = chainer.Variable(t_array)
model.cleargrads()
loss = model(x, t)
loss.backward()
loss.unchain_backward()
optimizer.update()
sum_loss += float(model.loss.data) * len(t.data)
sum_accuracy += float(model.accuracy.data) * len(t.data)
if train_iter.is_new_epoch:
train_loss = sum_loss / train_count
train_acc = sum_accuracy / train_count
# evaluation
sum_accuracy = 0
sum_loss = 0
chainer.config.train = False
chainer.config.enable_backprop = False
for batch in test_iter:
x_array, t_array = convert.concat_examples(batch, args.gpu)
x = chainer.Variable(x_array)
t = chainer.Variable(t_array)
loss = model(x, t)
sum_loss += float(loss.data) * len(t.data)
sum_accuracy += float(model.accuracy.data) * len(t.data)
test_iter.reset()
chainer.config.train = True
chainer.config.enable_backprop = True
test_loss = sum_loss / test_count
test_acc = sum_accuracy / test_count
message_str = "Epoch {}: train loss={:.4f}, acc={:.4f}, test loss={:.4f}, acc={:.4f}, elapsed={}"
logger.info(
message_str.format(train_iter.epoch, train_loss, train_acc,
test_loss, test_acc,
time.time() - st))
st = time.time()
sum_accuracy = 0
sum_loss = 0
if not args.debug:
serializers.save_npz(
os.path.join(save_dir,
"model_ep_{}.npz".format(train_iter.epoch)),
model)
iter_cnt += 1
def main():
parser = argparse.ArgumentParser(description='Chainer CIFAR example:')
parser.add_argument('--model', default='c3f2')
parser.add_argument('--batchsize', '-b', type=int, default=64)
parser.add_argument('--gpu', '-g', type=int, default=0)
parser.add_argument('--N', type=int, default=9)
parser.add_argument('--k', type=int, default=10)
parser.add_argument('--nb_valid', type=int, default=10000)
parser.add_argument('--seed', type=int, default=1701)
parser.add_argument('--out', '-o', default='result')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--test', action='store_true')
parser.add_argument('--resume', '-r', default='')
args = parser.parse_args()
start = time.time()
logger.initialize("evals_" + args.model)
logger.info(vars(args))
np.random.seed(args.seed)
save_dir = logger.get_savedir()
logger.info("Written to {}".format(save_dir))
logger.info('GPU: {}'.format(args.gpu))
logger.info('# Minibatch-size: {}'.format(args.batchsize))
logger.info('')
train_all, test = get_cifar10()
if args.debug:
valid = train_all[200:400]
batchsize = 20
else:
valid_choice = np.random.choice(range(len(train_all)),
args.nb_valid,
replace=False)
valid = [x for idx, x in enumerate(train_all) if idx in valid_choice]
batchsize = args.batchsize
valid_cnt = len(valid)
print(valid_cnt)
model = get_model(args.model, args.gpu, args.resume)
valid_iter = chainer.iterators.SerialIterator(valid,
batchsize,
repeat=False,
shuffle=False)
sum_accuracy = 0
sum_loss = 0
chainer.config.train = False
chainer.config.enable_backprop = False
predictions = []
for idx, batch in enumerate(valid_iter):
x_array, t_array = convert.concat_examples(batch, args.gpu)
x = chainer.Variable(x_array)
t = chainer.Variable(t_array)
loss = model(x, t)
sum_loss += float(loss.data) * len(t.data)
sum_accuracy += float(model.accuracy.data) * len(t.data)
predictions.extend(
np.argmax(cuda.to_cpu(model.pred.data), axis=1).tolist())
valid_iter.reset()
valid_loss = sum_loss / valid_cnt
valid_acc = sum_accuracy / valid_cnt
message_str = "Valid loss={:.4f}, acc={:.4f}"
logger.info(message_str.format(valid_loss, valid_acc))
canvas = plot_error([x[0] for x in valid], [x[1] for x in valid],
predictions)
cv2.imwrite("error.jpg", canvas)
print(time.time() - start)
def main():
parser = argparse.ArgumentParser(description='Chainer CIFAR example:')
parser.add_argument('resume')
parser.add_argument('--nb_trials', type=int, default=50)
parser.add_argument('--model', default='c5')
parser.add_argument('--gpu', '-g', type=int, default=0)
parser.add_argument('--nb_valid', type=int, default=10000)
parser.add_argument('--seed', type=int, default=1701)
parser.add_argument('--debug', action='store_true')
args = parser.parse_args()
start = time.time()
logger.initialize("grad_"+args.model)
logger.info(vars(args))
np.random.seed(args.seed)
save_dir = logger.get_savedir()
logger.info("Written to {}".format(save_dir))
logger.info('GPU: {}'.format(args.gpu))
train_all, test = get_cifar10()
if args.debug:
valid = train_all[200:400]
else:
valid_choice = np.random.choice(range(len(train_all)),
args.nb_valid, replace=False)
valid = [x for idx, x in enumerate(train_all) if idx in valid_choice]
print(len(valid))
model = get_model(args.model, args.gpu, args.resume)
# Get one image per iteration
valid_iter = chainer.iterators.SerialIterator(
valid, 1, repeat=False, shuffle=False)
if not os.path.exists("grads"):
os.makedirs("grads")
chainer.config.train = False
chainer.config.enable_backprop = True
for idx, tup in enumerate(valid_iter):
print(idx)
img = tup[0][0]
# Tile image to calculate all the trials at once
inp = np.tile(img.copy()[np.newaxis, ...], (args.nb_trials, 1, 1, 1))
label = tup[0][1][np.newaxis, ...]
sigma = (inp.max() - inp.min()) * 0.025 # noise level
model.cleargrads()
inp = inp + np.random.randn(*inp.shape).astype(np.float32) * sigma # Add noise to every image
x = Variable(cuda.to_gpu(inp, args.gpu))
xp = cuda.get_array_module(x)
pred = model.get_feature(x, False)
# print(class_list[int(cuda.to_cpu(xp.argmax(pred.data)))], class_list[int(label)])
pred.grad = xp.ones(pred.shape, dtype=np.float32)
pred.backward()
mean_grad = cuda.to_cpu(xp.mean(x.grad.copy(), axis=0))
mean_grad = np.max(np.abs(mean_grad), axis=0)
mean_grad = color.gray2rgb(mean_grad)
mean_grad = clip_image(mean_grad)
orig_img = np.transpose(img, (1, 2, 0))
masked = orig_img * mean_grad
out = np.concatenate((mean_grad, masked, orig_img), axis=1)
plt.imsave("grads/{:05d}.png".format(idx), out)
model.cleargrads()
print(time.time()-start)
def one_stage_training_gauss(args, model_load):
training_type = '_gauss_'
output_dir = './run/experiments/models'
add_to_file_path, model_save = create_file_name(args['dataset_name'],
args['model_name'],
args['im_size'],
training_type, output_dir)
num_iter = 1
num_epoch = 100
args['num_epoch'] = num_epoch
args['use_fixed_features'] = False
args['mean_requires_grad'] = True
args['learning_rate'] = 0.00001 #Max metric score: 0.8738366961479187
# args['mean'] = torch.rand(args['num_classes'], args['num_features']) # value is not important, for inititalization
# args['var'] = torch.rand(args['num_classes'], args['num_features']) # value is not important, for inititalization
args['mean'] = torch.zeros(
args['num_classes'],
args['num_features']) # value is not important, for inititalization
args['var'] = torch.ones(
args['num_classes'],
args['num_features']) # value is not important, for inititalization
args['class_prob'] = torch.zeros(
args['num_classes']) + 1 / args['num_classes']
_, args['loss_names'] = gauss_mixture_combined(args)
change_mean_from_gauss_first_iter = True
change_mean_from_gauss_other_iter = False
for iter in range(num_iter):
print('\nIteration: ', iter,
'****************************************')
print("Testing current model")
test_logs = T.test(args, model_load)
if (change_mean_from_gauss_first_iter and iter == 0):
mean, var, class_prob = compute_means_and_var(args, model_load)
#args['var'] = var
args['var'] = torch.zeros(
args['num_classes'], args['num_features']
) # value is not important, for inititalization
args['mean'] = mean
args['class_prob'] = class_prob
print("Test after changing mean before any iterations")
test_logs = T.test(args, model_load)
if iter > 0 and change_mean_from_gauss_other_iter:
mean, var, class_prob = compute_means_and_var(args, model_load)
args['var'] = var
args['mean'] = mean
args['class_prob'] = class_prob
pretrained_dict = torch.load(model_load)
model = get_model(args)
model_dict = model.state_dict()
# this ensures means and variances stay as specified in args[mean] and args[var]
del pretrained_dict['mean']
del pretrained_dict['sigma']
model_dict.update(pretrained_dict)
model.load_state_dict(model_dict)
torch.save(model.state_dict(), model_load)
print("Test after changing mean")
test_logs = T.test(args, model_load)
_, args['loss_names'] = gauss_mixture_combined(args)
T.train_normal(args, num_epoch, model_save, model_load)
model_load = model_save
if not args.use_labels:
args.clf_dir = os.path.join(args.model_dir, 'saved_classifier')
if not os.path.exists(args.clf_dir):
os.mkdir(args.clf_dir)
# Reset graph and set seed
tf.reset_default_graph()
print("Random seed: ", args.seed)
tf.random.set_random_seed(args.seed)
np.random.seed(args.seed)
### Load datasets
train_data, val_data, test_data = get_datasets(args)
### Get model
model = get_model(args, train_data)
elbo = model.loss
t_vars = tf.trainable_variables()
with tf.variable_scope(tf.get_variable_scope(), reuse=False):
optimizer = tf.train.AdamOptimizer(learning_rate=args.lr)
# Minimize elbo
grads_and_vars = optimizer.compute_gradients(-elbo, var_list=t_vars)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = optimizer.apply_gradients(grads_and_vars)
run_single_epoch = get_training_function(args.model_name)
# Initialize the variables (i.e. assign their default value)
init = tf.global_variables_initializer()
def main():
parser = argparse.ArgumentParser(description='Chainer CIFAR example:')
parser.add_argument('--model', default='c3f2')
parser.add_argument('--batchsize', '-b', type=int, default=64)
parser.add_argument('--learnrate', '-l', type=float, default=0.05)
parser.add_argument('--epoch', '-e', type=int, default=300)
parser.add_argument('--gpu', '-g', type=int, default=0)
parser.add_argument('--N', type=int, default=9)
parser.add_argument('--k', type=int, default=10)
parser.add_argument('--nb_valid', type=int, default=10000)
parser.add_argument('--seed', type=int, default=1701)
parser.add_argument('--out', '-o', default='result')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--test', action='store_true')
parser.add_argument('--resume', '-r', default='')
args = parser.parse_args()
start = time.time()
logger.initialize("outputs_" + args.model)
logger.info(vars(args))
np.random.seed(args.seed)
save_dir = logger.get_savedir()
logger.info("Written to {}".format(save_dir))
logger.info('GPU: {}'.format(args.gpu))
logger.info('# Minibatch-size: {}'.format(args.batchsize))
logger.info('# epoch: {}'.format(args.epoch))
logger.info('')
train_all, test = get_cifar10()
if args.debug:
train, valid, test = train_all[:200], train_all[200:400], test[:200]
batchsize = 20
else:
valid_choice = np.random.choice(range(len(train_all)),
args.nb_valid,
replace=False)
train = [
x for idx, x in enumerate(train_all) if idx not in valid_choice
]
valid = [x for idx, x in enumerate(train_all) if idx in valid_choice]
batchsize = args.batchsize
#import pdb;pdb.set_trace()
train_cnt, valid_cnt, test_cnt = len(train), len(valid), len(test)
print(train_cnt, valid_cnt, test_cnt)
model = get_model(args.model, args.gpu, args.resume)
optimizer = chainer.optimizers.MomentumSGD(args.learnrate)
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.WeightDecay(5e-4))
train_iter = chainer.iterators.SerialIterator(train, batchsize)
valid_iter = chainer.iterators.SerialIterator(valid,
batchsize,
repeat=False,
shuffle=False)
sum_accuracy = 0
sum_loss = 0
st = time.time()
iter_cnt = 0
chainer.config.train = True
chainer.config.enable_backprop = True
logger.info("Training...")
while train_iter.epoch < args.epoch:
batch = train_iter.next()
if train_iter.epoch % 60 == 0 and train_iter.is_new_epoch:
optimizer.lr *= 0.2
logger.info('Reducing learning rate to: {}'.format(optimizer.lr))
x_array, t_array = convert.concat_examples(batch, args.gpu)
x = chainer.Variable(x_array)
t = chainer.Variable(t_array)
model.cleargrads()
loss = model(x, t)
loss.backward()
loss.unchain_backward()
optimizer.update()
sum_loss += float(model.loss.data) * len(t.data)
sum_accuracy += float(model.accuracy.data) * len(t.data)
if train_iter.is_new_epoch:
train_loss = sum_loss / train_cnt
train_acc = sum_accuracy / train_cnt
# validation
sum_accuracy = 0
sum_loss = 0
chainer.config.train = False
chainer.config.enable_backprop = False
for batch in valid_iter:
x_array, t_array = convert.concat_examples(batch, args.gpu)
x = chainer.Variable(x_array)
t = chainer.Variable(t_array)
loss = model(x, t)
sum_loss += float(loss.data) * len(t.data)
sum_accuracy += float(model.accuracy.data) * len(t.data)
valid_iter.reset()
valid_loss = sum_loss / valid_cnt
valid_acc = sum_accuracy / valid_cnt
message_str = "Epoch {}: train loss={:.4f}, acc={:.4f}, valid loss={:.4f}, acc={:.4f}, elapsed={}"
logger.info(
message_str.format(train_iter.epoch, train_loss, train_acc,
valid_loss, valid_acc,
time.time() - st))
st = time.time()
chainer.config.train = True
chainer.config.enable_backprop = True
sum_accuracy = 0
sum_loss = 0
if not args.debug:
serializers.save_npz(
os.path.join(
save_dir,
"model_holdout_ep_{}.npz".format(train_iter.epoch)),
model)
iter_cnt += 1
if not test:
print(time.time() - start)
exit(1)
logger.info("Test...")
test_iter = chainer.iterators.SerialIterator(test,
batchsize,
repeat=False,
shuffle=False)
sum_accuracy = 0
sum_loss = 0
chainer.config.train = False
chainer.config.enable_backprop = False
st = time.time()
for batch in test_iter:
x_array, t_array = convert.concat_examples(batch, args.gpu)
x = chainer.Variable(x_array)
t = chainer.Variable(t_array)
loss = model(x, t)
sum_loss += float(loss.data) * len(t.data)
sum_accuracy += float(model.accuracy.data) * len(t.data)
test_loss = sum_loss / test_cnt
test_acc = sum_accuracy / test_cnt
message_str = "test loss={:.4f}, acc={:.4f}, elapsed={}"
logger.info(message_str.format(test_loss, test_acc, time.time() - st))
print(time.time() - start)