def test(args, model_load, imageSaver=None, EvaluatorIn=None): model = get_model(args) if model_load is not None: model.load_state_dict(torch.load(model_load), strict=False) logs = test_with_loaded_model(args, model, imageSaver, EvaluatorIn) return logs
def __init__(self, hparams: Namespace) -> None: super(MMClassifier, self).__init__() self.hparams = hparams self.model = get_model(self.hparams) if hparams.task_type == "multilabel": if hparams.weight_classes: freqs = [hparams.label_freqs[l] for l in hparams.labels] label_weights = (torch.FloatTensor(freqs) / hparams.train_data_len)**-1 self.criterion = nn.BCEWithLogitsLoss( pos_weight=label_weights.cuda()) else: self.criterion = nn.BCEWithLogitsLoss() else: self.criterion = nn.CrossEntropyLoss()
def train_normal(args, num_epoch, model_save=None, model_load=None, imgSaver=None, EvaluatorIn=None): # model_save is the name of the file name to save model to # model_load is the model to initialize training with if EvaluatorIn is None: evaluator = Evaluator(args) else: evaluator = EvaluatorIn(args) model = get_model(args) losses = get_losses(args) optimizer = get_optimizer(args, model) scheduler = get_scheduler(args, optimizer) valid_metric = None main_metric = evaluator.main_metric if model_load is not None: model.load_state_dict(torch.load(model_load), strict=False) train_dataset, valid_dataset = get_train_val_datasets(args) valid_logs, train_logs, best_loss_valid_metric = train(model, optimizer, scheduler, losses, train_dataset, valid_dataset, num_epoch, args, evaluator, imgSaver) if args['valid_dataset'] and args['train_with_metric']: valid_metric = valid_logs['metrics'][main_metric] if args['save_best_loss_name'] is not None or args['save_on_best_total_training_loss']: valid_metric = best_loss_valid_metric if not args['valid_dataset']: valid_metric = train_logs['metrics'][main_metric] valid_logs = train_logs if model_save is not None: torch.save(model.state_dict(), model_save) return valid_logs, train_logs, valid_metric
def main(): parser = argparse.ArgumentParser(description='Chainer CIFAR example:') parser.add_argument('--model', default='c3f2') parser.add_argument('--batchsize', '-b', type=int, default=64) parser.add_argument('--learnrate', '-l', type=float, default=0.05) parser.add_argument('--epoch', '-e', type=int, default=300) parser.add_argument('--gpu', '-g', type=int, default=0) parser.add_argument('--N', type=int, default=9) parser.add_argument('--k', type=int, default=10) parser.add_argument('--out', '-o', default='result') parser.add_argument('--debug', action='store_true') parser.add_argument('--resume', '-r', default='') args = parser.parse_args() start = time.time() logger.initialize("outputs_" + args.model) logger.info(vars(args)) save_dir = logger.get_savedir() logger.info("Written to {}".format(save_dir)) logger.info('GPU: {}'.format(args.gpu)) logger.info('# Minibatch-size: {}'.format(args.batchsize)) logger.info('# epoch: {}'.format(args.epoch)) logger.info('') train, test = get_cifar10() if args.debug: train, test = train[:200], test[:200] train_count, test_count = len(train), len(test) model = get_model(args.model, args.gpu, args.resume) optimizer = chainer.optimizers.MomentumSGD(args.learnrate) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.WeightDecay(5e-4)) train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) sum_accuracy = 0 sum_loss = 0 st = time.time() iter_cnt = 0 chainer.config.train = True chainer.config.enable_backprop = True while train_iter.epoch < args.epoch: batch = train_iter.next() if train_iter.epoch % 60 == 0 and train_iter.is_new_epoch: optimizer.lr *= 0.2 logger.info('Reducing learning rate to: {}'.format(optimizer.lr)) x_array, t_array = convert.concat_examples(batch, args.gpu) x = chainer.Variable(x_array) t = chainer.Variable(t_array) model.cleargrads() loss = model(x, t) loss.backward() loss.unchain_backward() optimizer.update() sum_loss += float(model.loss.data) * len(t.data) sum_accuracy += float(model.accuracy.data) * len(t.data) if train_iter.is_new_epoch: train_loss = sum_loss / train_count train_acc = sum_accuracy / train_count # evaluation sum_accuracy = 0 sum_loss = 0 chainer.config.train = False chainer.config.enable_backprop = False for batch in test_iter: x_array, t_array = convert.concat_examples(batch, args.gpu) x = chainer.Variable(x_array) t = chainer.Variable(t_array) loss = model(x, t) sum_loss += float(loss.data) * len(t.data) sum_accuracy += float(model.accuracy.data) * len(t.data) test_iter.reset() chainer.config.train = True chainer.config.enable_backprop = True test_loss = sum_loss / test_count test_acc = sum_accuracy / test_count message_str = "Epoch {}: train loss={:.4f}, acc={:.4f}, test loss={:.4f}, acc={:.4f}, elapsed={}" logger.info( message_str.format(train_iter.epoch, train_loss, train_acc, test_loss, test_acc, time.time() - st)) st = time.time() sum_accuracy = 0 sum_loss = 0 if not args.debug: serializers.save_npz( os.path.join(save_dir, "model_ep_{}.npz".format(train_iter.epoch)), model) iter_cnt += 1
def main(): parser = argparse.ArgumentParser(description='Chainer CIFAR example:') parser.add_argument('--model', default='c3f2') parser.add_argument('--batchsize', '-b', type=int, default=64) parser.add_argument('--gpu', '-g', type=int, default=0) parser.add_argument('--N', type=int, default=9) parser.add_argument('--k', type=int, default=10) parser.add_argument('--nb_valid', type=int, default=10000) parser.add_argument('--seed', type=int, default=1701) parser.add_argument('--out', '-o', default='result') parser.add_argument('--debug', action='store_true') parser.add_argument('--test', action='store_true') parser.add_argument('--resume', '-r', default='') args = parser.parse_args() start = time.time() logger.initialize("evals_" + args.model) logger.info(vars(args)) np.random.seed(args.seed) save_dir = logger.get_savedir() logger.info("Written to {}".format(save_dir)) logger.info('GPU: {}'.format(args.gpu)) logger.info('# Minibatch-size: {}'.format(args.batchsize)) logger.info('') train_all, test = get_cifar10() if args.debug: valid = train_all[200:400] batchsize = 20 else: valid_choice = np.random.choice(range(len(train_all)), args.nb_valid, replace=False) valid = [x for idx, x in enumerate(train_all) if idx in valid_choice] batchsize = args.batchsize valid_cnt = len(valid) print(valid_cnt) model = get_model(args.model, args.gpu, args.resume) valid_iter = chainer.iterators.SerialIterator(valid, batchsize, repeat=False, shuffle=False) sum_accuracy = 0 sum_loss = 0 chainer.config.train = False chainer.config.enable_backprop = False predictions = [] for idx, batch in enumerate(valid_iter): x_array, t_array = convert.concat_examples(batch, args.gpu) x = chainer.Variable(x_array) t = chainer.Variable(t_array) loss = model(x, t) sum_loss += float(loss.data) * len(t.data) sum_accuracy += float(model.accuracy.data) * len(t.data) predictions.extend( np.argmax(cuda.to_cpu(model.pred.data), axis=1).tolist()) valid_iter.reset() valid_loss = sum_loss / valid_cnt valid_acc = sum_accuracy / valid_cnt message_str = "Valid loss={:.4f}, acc={:.4f}" logger.info(message_str.format(valid_loss, valid_acc)) canvas = plot_error([x[0] for x in valid], [x[1] for x in valid], predictions) cv2.imwrite("error.jpg", canvas) print(time.time() - start)
def main(): parser = argparse.ArgumentParser(description='Chainer CIFAR example:') parser.add_argument('resume') parser.add_argument('--nb_trials', type=int, default=50) parser.add_argument('--model', default='c5') parser.add_argument('--gpu', '-g', type=int, default=0) parser.add_argument('--nb_valid', type=int, default=10000) parser.add_argument('--seed', type=int, default=1701) parser.add_argument('--debug', action='store_true') args = parser.parse_args() start = time.time() logger.initialize("grad_"+args.model) logger.info(vars(args)) np.random.seed(args.seed) save_dir = logger.get_savedir() logger.info("Written to {}".format(save_dir)) logger.info('GPU: {}'.format(args.gpu)) train_all, test = get_cifar10() if args.debug: valid = train_all[200:400] else: valid_choice = np.random.choice(range(len(train_all)), args.nb_valid, replace=False) valid = [x for idx, x in enumerate(train_all) if idx in valid_choice] print(len(valid)) model = get_model(args.model, args.gpu, args.resume) # Get one image per iteration valid_iter = chainer.iterators.SerialIterator( valid, 1, repeat=False, shuffle=False) if not os.path.exists("grads"): os.makedirs("grads") chainer.config.train = False chainer.config.enable_backprop = True for idx, tup in enumerate(valid_iter): print(idx) img = tup[0][0] # Tile image to calculate all the trials at once inp = np.tile(img.copy()[np.newaxis, ...], (args.nb_trials, 1, 1, 1)) label = tup[0][1][np.newaxis, ...] sigma = (inp.max() - inp.min()) * 0.025 # noise level model.cleargrads() inp = inp + np.random.randn(*inp.shape).astype(np.float32) * sigma # Add noise to every image x = Variable(cuda.to_gpu(inp, args.gpu)) xp = cuda.get_array_module(x) pred = model.get_feature(x, False) # print(class_list[int(cuda.to_cpu(xp.argmax(pred.data)))], class_list[int(label)]) pred.grad = xp.ones(pred.shape, dtype=np.float32) pred.backward() mean_grad = cuda.to_cpu(xp.mean(x.grad.copy(), axis=0)) mean_grad = np.max(np.abs(mean_grad), axis=0) mean_grad = color.gray2rgb(mean_grad) mean_grad = clip_image(mean_grad) orig_img = np.transpose(img, (1, 2, 0)) masked = orig_img * mean_grad out = np.concatenate((mean_grad, masked, orig_img), axis=1) plt.imsave("grads/{:05d}.png".format(idx), out) model.cleargrads() print(time.time()-start)
def one_stage_training_gauss(args, model_load): training_type = '_gauss_' output_dir = './run/experiments/models' add_to_file_path, model_save = create_file_name(args['dataset_name'], args['model_name'], args['im_size'], training_type, output_dir) num_iter = 1 num_epoch = 100 args['num_epoch'] = num_epoch args['use_fixed_features'] = False args['mean_requires_grad'] = True args['learning_rate'] = 0.00001 #Max metric score: 0.8738366961479187 # args['mean'] = torch.rand(args['num_classes'], args['num_features']) # value is not important, for inititalization # args['var'] = torch.rand(args['num_classes'], args['num_features']) # value is not important, for inititalization args['mean'] = torch.zeros( args['num_classes'], args['num_features']) # value is not important, for inititalization args['var'] = torch.ones( args['num_classes'], args['num_features']) # value is not important, for inititalization args['class_prob'] = torch.zeros( args['num_classes']) + 1 / args['num_classes'] _, args['loss_names'] = gauss_mixture_combined(args) change_mean_from_gauss_first_iter = True change_mean_from_gauss_other_iter = False for iter in range(num_iter): print('\nIteration: ', iter, '****************************************') print("Testing current model") test_logs = T.test(args, model_load) if (change_mean_from_gauss_first_iter and iter == 0): mean, var, class_prob = compute_means_and_var(args, model_load) #args['var'] = var args['var'] = torch.zeros( args['num_classes'], args['num_features'] ) # value is not important, for inititalization args['mean'] = mean args['class_prob'] = class_prob print("Test after changing mean before any iterations") test_logs = T.test(args, model_load) if iter > 0 and change_mean_from_gauss_other_iter: mean, var, class_prob = compute_means_and_var(args, model_load) args['var'] = var args['mean'] = mean args['class_prob'] = class_prob pretrained_dict = torch.load(model_load) model = get_model(args) model_dict = model.state_dict() # this ensures means and variances stay as specified in args[mean] and args[var] del pretrained_dict['mean'] del pretrained_dict['sigma'] model_dict.update(pretrained_dict) model.load_state_dict(model_dict) torch.save(model.state_dict(), model_load) print("Test after changing mean") test_logs = T.test(args, model_load) _, args['loss_names'] = gauss_mixture_combined(args) T.train_normal(args, num_epoch, model_save, model_load) model_load = model_save
if not args.use_labels: args.clf_dir = os.path.join(args.model_dir, 'saved_classifier') if not os.path.exists(args.clf_dir): os.mkdir(args.clf_dir) # Reset graph and set seed tf.reset_default_graph() print("Random seed: ", args.seed) tf.random.set_random_seed(args.seed) np.random.seed(args.seed) ### Load datasets train_data, val_data, test_data = get_datasets(args) ### Get model model = get_model(args, train_data) elbo = model.loss t_vars = tf.trainable_variables() with tf.variable_scope(tf.get_variable_scope(), reuse=False): optimizer = tf.train.AdamOptimizer(learning_rate=args.lr) # Minimize elbo grads_and_vars = optimizer.compute_gradients(-elbo, var_list=t_vars) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): train_op = optimizer.apply_gradients(grads_and_vars) run_single_epoch = get_training_function(args.model_name) # Initialize the variables (i.e. assign their default value) init = tf.global_variables_initializer()
def main(): parser = argparse.ArgumentParser(description='Chainer CIFAR example:') parser.add_argument('--model', default='c3f2') parser.add_argument('--batchsize', '-b', type=int, default=64) parser.add_argument('--learnrate', '-l', type=float, default=0.05) parser.add_argument('--epoch', '-e', type=int, default=300) parser.add_argument('--gpu', '-g', type=int, default=0) parser.add_argument('--N', type=int, default=9) parser.add_argument('--k', type=int, default=10) parser.add_argument('--nb_valid', type=int, default=10000) parser.add_argument('--seed', type=int, default=1701) parser.add_argument('--out', '-o', default='result') parser.add_argument('--debug', action='store_true') parser.add_argument('--test', action='store_true') parser.add_argument('--resume', '-r', default='') args = parser.parse_args() start = time.time() logger.initialize("outputs_" + args.model) logger.info(vars(args)) np.random.seed(args.seed) save_dir = logger.get_savedir() logger.info("Written to {}".format(save_dir)) logger.info('GPU: {}'.format(args.gpu)) logger.info('# Minibatch-size: {}'.format(args.batchsize)) logger.info('# epoch: {}'.format(args.epoch)) logger.info('') train_all, test = get_cifar10() if args.debug: train, valid, test = train_all[:200], train_all[200:400], test[:200] batchsize = 20 else: valid_choice = np.random.choice(range(len(train_all)), args.nb_valid, replace=False) train = [ x for idx, x in enumerate(train_all) if idx not in valid_choice ] valid = [x for idx, x in enumerate(train_all) if idx in valid_choice] batchsize = args.batchsize #import pdb;pdb.set_trace() train_cnt, valid_cnt, test_cnt = len(train), len(valid), len(test) print(train_cnt, valid_cnt, test_cnt) model = get_model(args.model, args.gpu, args.resume) optimizer = chainer.optimizers.MomentumSGD(args.learnrate) optimizer.setup(model) optimizer.add_hook(chainer.optimizer.WeightDecay(5e-4)) train_iter = chainer.iterators.SerialIterator(train, batchsize) valid_iter = chainer.iterators.SerialIterator(valid, batchsize, repeat=False, shuffle=False) sum_accuracy = 0 sum_loss = 0 st = time.time() iter_cnt = 0 chainer.config.train = True chainer.config.enable_backprop = True logger.info("Training...") while train_iter.epoch < args.epoch: batch = train_iter.next() if train_iter.epoch % 60 == 0 and train_iter.is_new_epoch: optimizer.lr *= 0.2 logger.info('Reducing learning rate to: {}'.format(optimizer.lr)) x_array, t_array = convert.concat_examples(batch, args.gpu) x = chainer.Variable(x_array) t = chainer.Variable(t_array) model.cleargrads() loss = model(x, t) loss.backward() loss.unchain_backward() optimizer.update() sum_loss += float(model.loss.data) * len(t.data) sum_accuracy += float(model.accuracy.data) * len(t.data) if train_iter.is_new_epoch: train_loss = sum_loss / train_cnt train_acc = sum_accuracy / train_cnt # validation sum_accuracy = 0 sum_loss = 0 chainer.config.train = False chainer.config.enable_backprop = False for batch in valid_iter: x_array, t_array = convert.concat_examples(batch, args.gpu) x = chainer.Variable(x_array) t = chainer.Variable(t_array) loss = model(x, t) sum_loss += float(loss.data) * len(t.data) sum_accuracy += float(model.accuracy.data) * len(t.data) valid_iter.reset() valid_loss = sum_loss / valid_cnt valid_acc = sum_accuracy / valid_cnt message_str = "Epoch {}: train loss={:.4f}, acc={:.4f}, valid loss={:.4f}, acc={:.4f}, elapsed={}" logger.info( message_str.format(train_iter.epoch, train_loss, train_acc, valid_loss, valid_acc, time.time() - st)) st = time.time() chainer.config.train = True chainer.config.enable_backprop = True sum_accuracy = 0 sum_loss = 0 if not args.debug: serializers.save_npz( os.path.join( save_dir, "model_holdout_ep_{}.npz".format(train_iter.epoch)), model) iter_cnt += 1 if not test: print(time.time() - start) exit(1) logger.info("Test...") test_iter = chainer.iterators.SerialIterator(test, batchsize, repeat=False, shuffle=False) sum_accuracy = 0 sum_loss = 0 chainer.config.train = False chainer.config.enable_backprop = False st = time.time() for batch in test_iter: x_array, t_array = convert.concat_examples(batch, args.gpu) x = chainer.Variable(x_array) t = chainer.Variable(t_array) loss = model(x, t) sum_loss += float(loss.data) * len(t.data) sum_accuracy += float(model.accuracy.data) * len(t.data) test_loss = sum_loss / test_cnt test_acc = sum_accuracy / test_cnt message_str = "test loss={:.4f}, acc={:.4f}, elapsed={}" logger.info(message_str.format(test_loss, test_acc, time.time() - st)) print(time.time() - start)