def main(): # image shape: N x H x W, pixel [0, 255] # label shape: N x 10 with np.load('mnist.npz', allow_pickle=True) as f: x_train, y_train = f['x_train'], f['y_train'] x_test, y_test = f['x_test'], f['y_test'] plt.imshow(x_train[59999], cmap='gray') plt.show() print(x_train.shape, x_train[0].max(), x_train[0].min()) #(60000, 28, 28) 255 0 5 print(x_test.shape, x_test[0].max(), x_test[0].min()) #(10000, 28, 28) 255 0 7 x_train = normalize_image(x_train) x_test = normalize_image(x_test) y_train = one_hot_labels(y_train) y_test = one_hot_labels(y_test) net = LeNet() net.fit(x_train, y_train, x_test, y_test, epoches=10, batch_size=16, lr=1e-3) accu = net.evaluate(x_test, labels=y_test) print("final accuracy {}".format(accu))
def main(): train_loader = torch.utils.data.DataLoader(datasets.MNIST( './data', train=True, download=True, transform=transforms.Compose([transforms.ToTensor()])), batch_size=256, shuffle=True) test_loader = torch.utils.data.DataLoader(datasets.MNIST( './data', train=False, download=True, transform=transforms.Compose([ transforms.ToTensor(), ])), batch_size=9, shuffle=True) test_batch = None for x in test_loader: test_batch = x[0] break # Use cpu if no cuda available device = torch.device("cuda") #device = torch.device("cpu") model = LeNet().to(device) optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9) train_with_logging(model, device, train_loader, optimizer, 200, 5, test_batch)
def main(): # image shape: N x H x W, pixel [0, 255] # label shape: N x 10 with np.load('mnist.npz', allow_pickle=True) as f: x_train, y_train = f['x_train'], f['y_train'] x_test, y_test = f['x_test'], f['y_test'] print(x_train.shape, x_train[0].max(), x_train[0].min()) #(60000, 28, 28) 255 0 5 print(x_test.shape, x_test[0].max(), x_test[0].min()) #(10000, 28, 28) 255 0 7 x_train = normalize_image(x_train) x_test = normalize_image(x_test) y_train = one_hot_labels(y_train) y_test = one_hot_labels(y_test) lr = 1.5e-4 batch_size = 8 net = LeNet() avgtime, accuracy = \ net.fit(x_train, y_train, x_test, y_test, epoches=5, batch_size=batch_size, lr=lr) accu = net.evaluate(x_test, labels=y_test) print('avgtime: ', avgtime) #print('accuracy: ', accuracy) ''' plt.plot(accuracy) plt.savefig('lr={}.jpg'.format(lr)) plt.show() ''' print("final accuracy {}".format(accu))
def LeNet_test(): # initializing the network network = LeNet(BATCH_SIZE) network.getTheParas(MODEL_FILE) # load the test data _, _, test_imgs, _, _, test_label = util.load_data(MNIST_PATH, False) log_string('------------start test-----------') num_batch = test_imgs.shape[0] // BATCH_SIZE start = 0 end = start + BATCH_SIZE loss = 0.0 total_correct = 0.0 total_seen = 0 for n in range(num_batch): log_string('--------{}/{}(batchs) completed!'.format(n + 1, num_batch)) current_img = test_imgs[start:end, ...] current_label = test_label[start:end, ...] start = end end += BATCH_SIZE predict_val, loss_val = network.forward(current_img, current_label) correct = np.sum(predict_val == current_label) total_correct += correct loss += loss_val total_seen += BATCH_SIZE log_string('eval mean loss: {}'.format(loss / num_batch)) log_string('eval accuracy: {}'.format(total_correct / total_seen))
def main(): parser = argparse.ArgumentParser() parser.add_argument( '--train-file', required=True, help=( 'Training dataset, as per the format from Kaggle: ' 'https://www.kaggle.com/c/digit-recognizer/data?select=train.csv')) parser.add_argument( '--test-file', required=True, help=( 'Testing dataset, as per the format from Kaggle: ' 'https://www.kaggle.com/c/digit-recognizer/data?select=test.csv')) args = parser.parse_args() train_df = pd.read_csv(args.train_file) test_df = pd.read_csv(args.test_file) train_X = train_df.drop(columns=['label']).values train_y = train_df['label'].values test_X = test_df.values model = LeNet() print(model.classifier) name = 'lenet_relu' model.train_on_dataset(train_X, train_y, save_path=name) test_predictions = model.predict_on_dataset(test_X) pred_df = pd.DataFrame({ 'ImageId': np.arange(1, test_X.shape[0] + 1), 'Label': test_predictions }) pred_df.to_csv(f'{name}_predictions.csv', index=False)
def __init__(self, img_shape=(160,320,3), model_file="lenet.h5", prev_model=None, batch_size=128, epochs=5): # net = NvidiaNet() net = LeNet() self.nnModel = net.network(img_shape=img_shape) self.modelLoaded = False self.modelFile = model_file self.prevModel = prev_model self.batchSize = batch_size self.epochs = epochs
def LeNet_train(): # initializing the network network = LeNet(BATCH_SIZE) # load the data train_imgs, val_imgs, _, train_label, val_label, _ = util.load_data( MNIST_PATH) for epoch in range(MAX_EPOCH): eval_one_epoch(network, val_imgs, val_label) log_string('------------start train {}/{}----------'.format( epoch, MAX_EPOCH)) train_one_epoch(network, train_imgs, train_label, epoch)
def inference(): # initializing the network network = LeNet(BATCH_SIZE) network.getTheParas(MODEL_FILE) print(IMAGE_PATH) image_paths = glob.glob(os.path.join(IMAGE_PATH, '*')) for image_path in image_paths: image_data = cv2.imread(image_path, 0) image_data = image_data[newaxis, :, :, newaxis] predict_val = network.inference(image_data) print(image_path, ':', predict_val[0][0])
def aiTest(images, shape): model = LeNet() y_test = [] x_test = images generate_images = [] for image in x_test: confidence = model.predict(image)[0] predicted_class = np.argmax(confidence) y_test.append(predicted_class) attacker = PixelAttacker((x_test, y_test)) for i in range(len(x_test)): generate_images.append(attacker.attack(i, model, verbose=False)[10]) return generate_images
def test_image(filename, num_class, weights_path='Default'): img_string = tf.gfile.FastGFile(filename, 'rb').read() img_decoded = tf.image.decode_jpeg(img_string, channels=channels) img_resized = tf.image.resize_images(img_decoded, [image_size, image_size]) img_reshape = tf.reshape(img_resized, shape=[1, image_size, image_size, channels]) model = LeNet(img_reshape, keep_prob, num_classes, batch_size, image_size, channels) score = tf.nn.softmax(model.fc6) max = tf.argmax(score, 1) saver = tf.train.Saver() with tf.Session() as sess: sess.run(tf.global_variables_initializer()) saver.restore(sess, "./tmp/checkpoint/model_epoch10.ckpt") print(sess.run(model.fc6)) prob = sess.run(max)[0] print("The number is %s" % (class_name[prob]))
def main(): batch_size = 128 epoch = 15 data = DATA() model = LeNet(data.input_shape, data.num_classes) hist = model.fit(data.x_train, data.y_train, batch_size=batch_size, epochs=epoch, validation_split=0.2) score = model.evaluate(data.x_test, data.y_test, batch_size=batch_size) print() print('Test Loss= ', score) plot_loss(hist) plt.show()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--model-file', required=True, help='File path to a trained PyTorch model') parser.add_argument( '--target-digit', required=True, type=int, choices=range(10), help='The digit that we want to find the \"epitome\" for') parser.add_argument('--num-samples', type=int, default=10, help=('The number of epitome samples to generate')) parser.add_argument( '--use-training-file', default=None, help= ('Start constructions with entries from the given training set, ' 'rather than from uniform noise. The format of the file should match: ' 'https://www.kaggle.com/c/digit-recognizer/data?select=train.csv')) args = parser.parse_args() model = LeNet() model.load_saved_state(args.model_file) starter_X = None if args.use_training_file: train_df = pd.read_csv(args.use_training_file) starter_df = train_df[train_df['label'] == args.target_digit] starter_X = starter_df.drop( columns=['label']).values[:args.num_samples] opt_inputs = model.optimize_for_digit( args.target_digit, num_samples=args.num_samples, starter_X=starter_X, ) for i in range(opt_inputs.shape[0]): img = Image.fromarray(opt_inputs[i], 'L') img.save(f'digit_{args.target_digit}_{i}.png')
def get_model(self): if self.exp_type == 'pnml_cifar10': model = load_pretrained_resnet20_cifar10_model(resnet20()) elif self.exp_type == 'random_labels': model = WideResNet() elif self.exp_type == 'out_of_dist_svhn': model = load_pretrained_resnet20_cifar10_model(resnet20()) elif self.exp_type == 'out_of_dist_noise': model = load_pretrained_resnet20_cifar10_model(resnet20()) elif self.exp_type == 'pnml_mnist': model = Net() elif self.exp_type == 'adversarial': model = load_pretrained_resnet20_cifar10_model(resnet20()) elif self.exp_type == 'pnml_cifar10_lenet': model = LeNet() # VGG('VGG16') else: raise NameError('No experiment type: %s' % self.exp_type) return model
def forward_pytorch(weightfile, image): #net=resnet.resnet18() #net = resnet.resnet18() net=LeNet(1,2) checkpoint = torch.load(weightfile) net.load_state_dict(checkpoint['weight']) net.double() # to double if args.cuda: net.cuda() print(net) net.eval() image = torch.from_numpy(image.astype(np.float64)) # to double if args.cuda: image = Variable(image.cuda()) else: image = Variable(image) t0 = time.time() blobs = net.forward(image) print(blobs.data.numpy().flatten()) t1 = time.time() return t1-t0, blobs, net, torch.from_numpy(blobs.data.numpy())
def torch_setup(self): self.data_train = MNIST('./data/mnist', train=True, download=True, transform=self.compose) self.data_test = MNIST('./data/mnist', train=False, download=True, transform=self.compose) self.data_train_loader = DataLoader(self.data_train, batch_size=self.batch_size, shuffle=True, num_workers=4) self.data_test_loader = DataLoader(self.data_test, batch_size=self.batch_size, num_workers=4) self.criterion = nn.CrossEntropyLoss() self.net = LeNet() self.optimizer = getattr(optim, self.opt)(self.net.parameters(), lr=self.learning_rate) try: self.net.load_state_dict(load('model_params.pkl')) self.loaded = True print("Loaded") except Exception as e: print(e) self.loaded = False print("Not loaded")
def main(): global args, rank, world_size, best_prec1, dataset_len if args.dist == 1: rank, world_size = dist_init() else: rank = 0 world_size = 1 model = LeNet() model.cuda() param_copy = [ param.clone().type(torch.cuda.FloatTensor).detach() for param in model.parameters() ] for param in param_copy: param.requires_grad = True if args.dist == 1: model = DistModule(model) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss() optimizer = torch.optim.SGD(param_copy, args.base_lr, momentum=args.momentum, weight_decay=args.weight_decay) # optionally resume from a checkpoint last_iter = -1 # Data loading code train_dataset = datasets.MNIST(root='./data', train=True, transform=transforms.ToTensor(), download=False) val_dataset = datasets.MNIST(root='./data', train=False, transform=transforms.ToTensor(), download=False) dataset_len = len(train_dataset) args.max_iter = math.ceil( (dataset_len * args.epoch) / (world_size * args.batch_size)) if args.dist == 1: train_sampler = DistributedGivenIterationSampler(train_dataset, args.max_iter, args.batch_size, last_iter=last_iter) val_sampler = DistributedSampler(val_dataset, round_up=False) else: train_sampler = DistributedGivenIterationSampler(train_dataset, args.max_iter, args.batch_size, world_size=1, rank=0, last_iter=last_iter) val_sampler = None # pin_memory if true, will copy the tensor to cuda pinned memory train_loader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True, sampler=train_sampler) val_loader = DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True, sampler=val_sampler) train(train_loader, val_loader, model, criterion, optimizer, param_copy)
# REMEMBER TO RIGHTLY SET THE FOLLOWING PATH TEST_DIR = 'test' RESULT_DIR = 'results' # Process test data and create batches in memory graph = tf.Graph() gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9) sess_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=True, gpu_options=gpu_options) sess_config.gpu_options.allow_growth = True sess = tf.Session(config=sess_config) if TYPE_OF_MODEL == 'LE-NET': config = Config(TYPE_OF_MODEL) model = LeNet(config, sess, graph) elif TYPE_OF_MODEL == 'DEEP-CONVNET': config = Config(TYPE_OF_MODEL) model = DeepConv(config, sess, graph) elif TYPE_OF_MODEL == 'ALEX-NET': config = Config(TYPE_OF_MODEL) model = AlexNet(config, sess, graph) model.restore() print(TYPE_OF_MODEL + " CNN Model Restored") test_batches = init_test_data(model.config) # Get the predictions and write them into a CSV file with open(RESULT_DIR + '/' + TYPE_OF_MODEL + '.csv', 'wb') as csvfile: writer = csv.writer(csvfile)
img_grid = vutils.make_grid(data_batch, nrow=4, normalize=True, scale_each=True) # img_grid = vutils.make_grid(data_batch, nrow=4, normalize=False, scale_each=False) writer.add_image("input img", img_grid, 0) writer.close() # ----------------------------------- 5 add_graph ----------------------------------- # flag = 0 flag = 1 if flag: writer = SummaryWriter(comment='test_your_comment', filename_suffix="_test_your_filename_suffix") # 模型 fake_img = torch.randn(1, 3, 32, 32) lenet = LeNet(classes=2) writer.add_graph(lenet, fake_img) writer.close() from torchsummary import summary print(summary(lenet, (3, 32, 32), device="cpu"))
def main(): # Training settings parser = argparse.ArgumentParser(description='PyTorch MNIST Example') parser.add_argument('--config_file', type=str, default='', help="config file") parser.add_argument('--stage', type=str, default='', help="select the pruning stage") args = parser.parse_args() config = Config(args) use_cuda = True torch.manual_seed(1) device = torch.device("cuda" if use_cuda else "cpu") kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} train_loader = torch.utils.data.DataLoader(datasets.MNIST( '../data', train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ])), batch_size=64, shuffle=True, **kwargs) test_loader = torch.utils.data.DataLoader(datasets.MNIST( '../data', train=False, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ])), batch_size=1000, shuffle=True, **kwargs) model = None if config.arch == 'lenet_bn': model = LeNet_BN().to(device) elif config.arch == 'lenet': model = LeNet().to(device) elif config.arch == 'lenet_adv': model = LeNet_adv(config.width_multiplier).to(device) torch.cuda.set_device(config.gpu) model.cuda(config.gpu) config.model = model ADMM = None config.prepare_pruning() if config.admm: ADMM = admm.ADMM(config) criterion = CrossEntropyLossMaybeSmooth(smooth_eps=config.smooth_eps).cuda( config.gpu) config.smooth = config.smooth_eps > 0.0 config.mixup = config.alpha > 0.0 config.warmup = (not config.admm) and config.warmup_epochs > 0 optimizer_init_lr = config.warmup_lr if config.warmup else config.lr if (config.optimizer == 'sgd'): optimizer = torch.optim.SGD(config.model.parameters(), optimizer_init_lr, momentum=0.9, weight_decay=1e-4) elif (config.optimizer == 'adam'): optimizer = torch.optim.Adam(config.model.parameters(), optimizer_init_lr) else: raise Exception("unknown optimizer") scheduler = None if config.lr_scheduler == 'cosine': scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=config.epochs * len(train_loader), eta_min=4e-08) elif config.lr_scheduler == 'default': pass else: raise Exception("unknown lr scheduler") if config.load_model: # unlike resume, load model does not care optimizer status or start_epoch print('==> Loading from {}'.format(config.load_model)) config.model.load_state_dict( torch.load(config.load_model, map_location={'cuda:0': 'cuda:{}'.format(config.gpu)})) test(config, device, test_loader) global best_acc if config.resume: if os.path.isfile(config.resume): checkpoint = torch.load(config.resume) config.start_epoch = checkpoint['epoch'] best_acc = checkpoint['best_acc'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded checkpoint '{}' (epoch {})".format( config.resume, checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(config.resume)) if config.masked_retrain: # make sure small weights are pruned and confirm the acc print("<============masking both weights and gradients for retrain") admm.masking(config) print("<============testing sparsity before retrain") admm.test_sparsity(config) test(config, device, test_loader) if config.masked_progressive: admm.zero_masking(config) for epoch in range(0, config.epochs + 1): train(config, ADMM, device, train_loader, criterion, optimizer, scheduler, epoch) test(config, device, test_loader) save_checkpoint( config, { 'epoch': epoch + 1, 'arch': config.arch, 'state_dict': config.model.state_dict(), 'best_acc': best_acc, 'optimizer': optimizer.state_dict() }) print('overall best_acc is {}'.format(best_acc)) if (config.save_model and config.admm): print('saving model {}'.format(config.save_model)) torch.save(config.model.state_dict(), config.save_model)
# prepare the train & test dataset and labels # expand 3D dataset to 4D dataset (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], x_train.shape[2], 1)) x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], x_test.shape[2], 1)) # convert the labels form integers to vectors lb = LabelBinarizer() y_train = lb.fit_transform(y_train) y_test = lb.fit_transform(y_test) # define the model lenet = LeNet() model = lenet.net # optimizer sgd = SGD(momentum=0.9, nesterov=True) print("[INFO] start training...") model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=["accuracy"]) H = model.fit(x_train, y_train, validation_data=(x_test, y_test), epochs=100, batch_size=128)
torchvision.datasets.ImageFolder(args.train, transform = data_transforms), batch_size=args.batch_size, shuffle=True, num_workers=args.worker ) val_loader = torch.utils.data.DataLoader( torchvision.datasets.ImageFolder(args.val, transform = data_transforms), batch_size=args.batch_size, shuffle=True, num_workers=args.worker ) return train_loader, val_loader train_loader, val_loader = get_dataloader() classes = 10 net = LeNet(classes) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.9) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") # model to gpu net.to(device) print("Start Training...") os.makedirs('./expr', exist_ok=True) for epoch in range(1000): net.train() running_loss = 0.0 for i, data in enumerate(train_loader): inputs, labels = data inputs, labels = inputs.to(device, dtype=torch.float), labels.to(device)
'resource_info')), 'Filename containing cluster information') tf.app.flags.DEFINE_integer('max_steps', 1000000, """Number of iterations to run for each workers.""") tf.app.flags.DEFINE_integer('log_frequency', 50, """How many steps between two logs.""") tf.app.flags.DEFINE_integer('batch_size', 32, """Batch size""") tf.app.flags.DEFINE_boolean('sync', True, '') mnist = input_data.read_data_sets('MNIST_data', one_hot=True) # Build single-GPU LeNet model single_gpu_graph = tf.Graph() with single_gpu_graph.as_default(): model = LeNet() parallax_config = parallax.Config() ckpt_config = parallax.CheckPointConfig(ckpt_dir='ckpt', save_ckpt_steps=FLAGS.log_frequency) parallax_config.ckpt_config = ckpt_config sess, num_workers, worker_id, num_replicas_per_worker = parallax.parallel_run( single_gpu_graph, FLAGS.resource_info_file, sync=FLAGS.sync, parallax_config=parallax_config) start = time.time() for i in range(FLAGS.max_steps): batch = mnist.train.next_batch(FLAGS.batch_size, shuffle=False)
#!/usr/bin/env python3 import torch as th import torchvision as tv import torch.nn as nn import torch.optim as optim from torch.autograd import Variable as V from torchvision import transforms from lenet import LeNet teacher = LeNet() teacher = teacher.cuda() transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) trainset = tv.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train)
"Allow device soft device placement") tf.flags.DEFINE_boolean("log_device_placement", False, "Log placement of ops on devices") FLAGS = tf.flags.FLAGS (x_train_val, y_train_val), ( x_test, y_test) = load_data() # training data: 50000, test data: 10000 x_train, y_train, x_test, y_test, x_val, y_val = dh.shuffle_data( x_train_val, y_train_val, x_test, y_test, FLAGS.num_classes) with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): lenet = LeNet(FLAGS) # Define Training procedure # * hint learning rate decay를 위한 operation을 통해 감쇠된 learning rate를 optimizer에 적용 learning_rate = FLAGS.starter_learning_rate # Learning rate decay가 적용되지 않는다면 Starter learning rate값이 그대로 전달됨 global_step = tf.Variable(0, name="global_step", trainable=False) # iteration 수 if FLAGS.learning_rate_decay_rate != 1: learning_rate = learning_rate * FLAGS.learning_rate_decay_rate**( global_step / FLAGS.learning_rate_decay_step) if FLAGS.optimizer == 'adam': optimizer = tf.train.AdamOptimizer( learning_rate=learning_rate) #Optimizer elif FLAGS.optimizer == 'adagrad': optimizer = tf.train.AdagradOptimizer( learning_rate=learning_rate)
def main(): parser = argparse.ArgumentParser() mode_group = parser.add_mutually_exclusive_group(required=True) mode_group.add_argument("--train", action="store_true", help="To train the network.") mode_group.add_argument("--test", action="store_true", help="To test the network.") parser.add_argument("--epochs", default=10, type=int, help="Desired number of epochs.") parser.add_argument("--dropout", action="store_true", help="Whether to use dropout or not.") parser.add_argument("--uncertainty", action="store_true", help="Use uncertainty or not.") parser.add_argument("--dataset", action="store_true", help="The dataset to use.") parser.add_argument("--outsample", action="store_true", help="Use out of sample test image") uncertainty_type_group = parser.add_mutually_exclusive_group() uncertainty_type_group.add_argument( "--mse", action="store_true", help= "Set this argument when using uncertainty. Sets loss function to Expected Mean Square Error." ) uncertainty_type_group.add_argument( "--digamma", action="store_true", help= "Set this argument when using uncertainty. Sets loss function to Expected Cross Entropy." ) uncertainty_type_group.add_argument( "--log", action="store_true", help= "Set this argument when using uncertainty. Sets loss function to Negative Log of the Expected Likelihood." ) dataset_type_group = parser.add_mutually_exclusive_group() dataset_type_group.add_argument( "--mnist", action="store_true", help="Set this argument when using MNIST dataset") dataset_type_group.add_argument( "--emnist", action="store_true", help="Set this argument when using EMNIST dataset") dataset_type_group.add_argument( "--CIFAR", action="store_true", help="Set this argument when using CIFAR dataset") dataset_type_group.add_argument( "--fmnist", action="store_true", help="Set this argument when using FMNIST dataset") args = parser.parse_args() if args.dataset: if args.mnist: from mnist import dataloaders, label_list elif args.CIFAR: from CIFAR import dataloaders, label_list elif args.fmnist: from fashionMNIST import dataloaders, label_list if args.train: num_epochs = args.epochs use_uncertainty = args.uncertainty num_classes = 10 model = LeNet(dropout=args.dropout) if use_uncertainty: if args.digamma: criterion = edl_digamma_loss elif args.log: criterion = edl_log_loss elif args.mse: criterion = edl_mse_loss else: parser.error( "--uncertainty requires --mse, --log or --digamma.") else: criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=1e-3, weight_decay=0.005) exp_lr_scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1) device = get_device() model = model.to(device) model, metrics = train_model(model, dataloaders, num_classes, criterion, optimizer, scheduler=exp_lr_scheduler, num_epochs=num_epochs, device=device, uncertainty=use_uncertainty) state = { "epoch": num_epochs, "model_state_dict": model.state_dict(), "optimizer_state_dict": optimizer.state_dict(), } if use_uncertainty: if args.digamma: torch.save(state, "./results/model_uncertainty_digamma.pt") print("Saved: ./results/model_uncertainty_digamma.pt") if args.log: torch.save(state, "./results/model_uncertainty_log.pt") print("Saved: ./results/model_uncertainty_log.pt") if args.mse: torch.save(state, "./results/model_uncertainty_mse.pt") print("Saved: ./results/model_uncertainty_mse.pt") else: torch.save(state, "./results/model.pt") print("Saved: ./results/model.pt") elif args.test: use_uncertainty = args.uncertainty device = get_device() model = LeNet() model = model.to(device) optimizer = optim.Adam(model.parameters()) if use_uncertainty: if args.digamma: checkpoint = torch.load( "./results/model_uncertainty_digamma.pt") if args.log: checkpoint = torch.load("./results/model_uncertainty_log.pt") if args.mse: checkpoint = torch.load("./results/model_uncertainty_mse.pt") else: checkpoint = torch.load("./results/model.pt") filename = "./results/rotate.jpg" model.load_state_dict(checkpoint["model_state_dict"]) optimizer.load_state_dict(checkpoint["optimizer_state_dict"]) model.eval() if args.outsample: img = Image.open("./data/arka.jpg").convert('L').resize((28, 28)) img = TF.to_tensor(img) img.unsqueeze_(0) else: a = iter(dataloaders['test']) img, label = next(a) rotating_image_classification(model, img, filename, label_list, uncertainty=use_uncertainty) img = transforms.ToPILImage()(img[0][0]) test_single_image(model, img, label_list, uncertainty=use_uncertainty)
tf.flags.DEFINE_boolean("log_device_placement", False, "Log placement of ops on devices") FLAGS = tf.flags.FLAGS (x_train_val, y_train_val), (x_test, y_test) = load_data() # training data: 50000, test data: 10000 x_train, y_train, x_test, y_test, x_val, y_val = dh.shuffle_data( x_train_val, y_train_val, x_test, y_test, FLAGS.num_classes) with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): lenet = LeNet( FLAGS) #LeNet 클래스의 인스턴스 생성 후 Hyperparameter가 정의돼 있는 FLAGS로 초기화 # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) # iteration 수 # * hint learning rate decay를 위한 operation을 통해 감쇠된 learning rate를 optimizer에 적용 # decayed_learning_rate = learning_rate * decay_rate ^ (global_step / decay_steps) # decayed_learning_rate = FLAGS.lr * FLAGS.lr_decay ** (global_step / FLAGS.lr_decay_steps) # optimizer = tf.train.AdamOptimizer(learning_rate=decayed_learning_rate) optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.lr) #Optimizer grads_and_vars = optimizer.compute_gradients(lenet.loss) # gradient 계산 train_op = optimizer.apply_gradients( grads_and_vars, global_step=global_step) # back-propagation # Output directory for models and summaries
#!/usr/bin/env python3 import torch as th import torchvision as tv import torch.nn as nn import torch.optim as optim from torch.autograd import Variable as V from torchvision import transforms from lenet import LeNet from sobolev import SobolevLoss USE_SOBOLEV = False student = LeNet() teacher = LeNet() teacher.load_state_dict(th.load('teacher.pth')) student = student.cuda() teacher = teacher.cuda() transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
def train_lenet(split_dir): """训练lenet""" set_seed() # 设置随机种子 rmb_label = {"1": 0, "100": 1} # 参数设置 MAX_EPOCH = 10 BATCH_SIZE = 16 LR = 0.01 log_interval = 10 val_interval = 1 """Step 1: 数据读取""" train_dir = os.path.join(split_dir, "train") valid_dir = os.path.join(split_dir, "valid") test_dir = os.path.join(split_dir, "test") norm_mean = [0.485, 0.456, 0.406] norm_std = [0.229, 0.224, 0.225] # 对训练数据进行变换,添加RandomCrop进行数据增强 train_transform = transforms.Compose([ transforms.Resize((32, 32)), transforms.RandomCrop(32, padding=4), transforms.ToTensor(), transforms.Normalize(norm_mean, norm_std), ]) # 对验证数据进行变换 valid_transform = transforms.Compose([ transforms.Resize((32, 32)), transforms.ToTensor(), transforms.Normalize(norm_mean, norm_std), ]) # 构建RMBDataset实例 train_data = RMBDataset(data_dir=train_dir, transform=train_transform) valid_data = RMBDataset(data_dir=valid_dir, transform=valid_transform) # 构建DataLoader train_loader = DataLoader(dataset=train_data, batch_size=BATCH_SIZE, shuffle=True) valid_loader = DataLoader(dataset=valid_data, batch_size=BATCH_SIZE) """Step 2: 模型""" net = LeNet(classes=2) net.initialize_weights() """Step 3: 损失函数""" criterion = nn.CrossEntropyLoss() """Step 4: 优化器""" optimizer = optim.SGD(net.parameters(), lr=LR, momentum=0.9) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1) """Step 5: 训练""" train_curve = [] valid_curve = [] figure_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'result') for epoch in range(MAX_EPOCH): loss_mean = 0. correct = 0. total = 0. net.train() for i, data in enumerate(train_loader): # forward inputs, labels = data outputs = net(inputs) # backward optimizer.zero_grad() loss = criterion(outputs, labels) loss.backward() # update weights optimizer.step() # 统计分类情况 _, predicted = torch.max(outputs.data, 1) total += labels.size(0) correct += (predicted == labels).squeeze().sum().numpy() # 打印训练信息 loss_mean += loss.item() train_curve.append(loss.item()) if (i + 1) % log_interval == 0: loss_mean = loss_mean / log_interval print( "Training:Epoch[{:0>3}/{:0>3}] Iteration[{:0>3}/{:0>3}] Loss: {:.4f} Acc:{:.2%}" .format(epoch, MAX_EPOCH, i + 1, len(train_loader), loss_mean, correct / total)) loss_mean = 0. scheduler.step() # 更新学习率 # validate the model if (epoch + 1) % val_interval == 0: correct_val = 0. total_val = 0. loss_val = 0. net.eval() with torch.no_grad(): for j, data in enumerate(valid_loader): inputs, labels = data outputs = net(inputs) loss = criterion(outputs, labels) _, predicted = torch.max(outputs.data, 1) total_val += labels.size(0) correct_val += ( predicted == labels).squeeze().sum().numpy() loss_val += loss.item() valid_curve.append(loss_val / valid_loader.__len__()) print( "Valid:\t Epoch[{:0>3}/{:0>3}] Iteration[{:0>3}/{:0>3}] Loss: {:.4f} Acc:{:.2%}" .format(epoch, MAX_EPOCH, j + 1, len(valid_loader), loss_val, correct_val / total_val)) train_x = range(len(train_curve)) train_y = train_curve train_iters = len(train_loader) valid_x = np.arange( 1, len(valid_curve) + 1 ) * train_iters * val_interval # 由于valid中记录的是epochloss,需要对记录点进行转换到iterations valid_y = valid_curve plt.plot(train_x, train_y, label='Train') plt.plot(valid_x, valid_y, label='Valid') plt.legend(loc='upper right') plt.ylabel('loss value') plt.xlabel('Iteration') figure_path = os.path.join(figure_dir, '0201.png') plt.savefig(figure_path) plt.close()
batch = get_batch(training_loader) _, _, height, width = batch['image'].shape shape = height, width shape = net_input_size, net_input_size # https://discuss.pytorch.org/t/how-can-l-load-my-best-model-as-a-feature-extractor-evaluator/17254/6 activation = {} def get_activation(name): def hook(model, input, output): activation[name] = output.detach() return hook # Actual network net = LeNet(shape) net.fc2.register_forward_hook(get_activation('fc2')) net.conv2.register_forward_hook(get_activation('conv2')) # Log architecture dummy_input = torch.zeros(1, 1, net_input_size, net_input_size) writer.add_graph( net, input_to_model=dummy_input, # verbose=True, ) # Loss criterion = nn.CrossEntropyLoss() # Optimizer
cuda = not args.disable_cuda and torch.cuda.is_available() trainset, testset, classes, shape = get_mnist(input_dimensions=2) w, h = shape trainloader = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size, shuffle=True, pin_memory=cuda) testloader = torch.utils.data.DataLoader(testset, batch_size=args.batch_size, shuffle=False, pin_memory=cuda) model = LeNet(k=len(classes), args=args) if cuda: model.cuda() for epoch in range(1, args.epochs + 1): model.train() for i, (x_batch, y_batch) in enumerate(trainloader): if cuda: x_batch, y_batch = x_batch.cuda(), y_batch.cuda() model.optimizer.zero_grad() # forward + backward + optimize output = model(x_batch)