def main():
# image shape: N x H x W, pixel [0, 255]
# label shape: N x 10
with np.load('mnist.npz', allow_pickle=True) as f:
x_train, y_train = f['x_train'], f['y_train']
x_test, y_test = f['x_test'], f['y_test']
plt.imshow(x_train[59999], cmap='gray')
plt.show()
print(x_train.shape, x_train[0].max(),
x_train[0].min()) #(60000, 28, 28) 255 0 5
print(x_test.shape, x_test[0].max(),
x_test[0].min()) #(10000, 28, 28) 255 0 7
x_train = normalize_image(x_train)
x_test = normalize_image(x_test)
y_train = one_hot_labels(y_train)
y_test = one_hot_labels(y_test)
net = LeNet()
net.fit(x_train,
y_train,
x_test,
y_test,
epoches=10,
batch_size=16,
lr=1e-3)
accu = net.evaluate(x_test, labels=y_test)
print("final accuracy {}".format(accu))
def main():
train_loader = torch.utils.data.DataLoader(datasets.MNIST(
'./data',
train=True,
download=True,
transform=transforms.Compose([transforms.ToTensor()])),
batch_size=256,
shuffle=True)
test_loader = torch.utils.data.DataLoader(datasets.MNIST(
'./data',
train=False,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
])),
batch_size=9,
shuffle=True)
test_batch = None
for x in test_loader:
test_batch = x[0]
break
# Use cpu if no cuda available
device = torch.device("cuda")
#device = torch.device("cpu")
model = LeNet().to(device)
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
train_with_logging(model, device, train_loader, optimizer, 200, 5,
test_batch)
def main():
# image shape: N x H x W, pixel [0, 255]
# label shape: N x 10
with np.load('mnist.npz', allow_pickle=True) as f:
x_train, y_train = f['x_train'], f['y_train']
x_test, y_test = f['x_test'], f['y_test']
print(x_train.shape, x_train[0].max(),
x_train[0].min()) #(60000, 28, 28) 255 0 5
print(x_test.shape, x_test[0].max(),
x_test[0].min()) #(10000, 28, 28) 255 0 7
x_train = normalize_image(x_train)
x_test = normalize_image(x_test)
y_train = one_hot_labels(y_train)
y_test = one_hot_labels(y_test)
lr = 1.5e-4
batch_size = 8
net = LeNet()
avgtime, accuracy = \
net.fit(x_train, y_train, x_test, y_test, epoches=5, batch_size=batch_size, lr=lr)
accu = net.evaluate(x_test, labels=y_test)
print('avgtime: ', avgtime)
#print('accuracy: ', accuracy)
'''
plt.plot(accuracy)
plt.savefig('lr={}.jpg'.format(lr))
plt.show()
'''
print("final accuracy {}".format(accu))
def LeNet_test():
# initializing the network
network = LeNet(BATCH_SIZE)
network.getTheParas(MODEL_FILE)
# load the test data
_, _, test_imgs, _, _, test_label = util.load_data(MNIST_PATH, False)
log_string('------------start test-----------')
num_batch = test_imgs.shape[0] // BATCH_SIZE
start = 0
end = start + BATCH_SIZE
loss = 0.0
total_correct = 0.0
total_seen = 0
for n in range(num_batch):
log_string('--------{}/{}(batchs) completed!'.format(n + 1, num_batch))
current_img = test_imgs[start:end, ...]
current_label = test_label[start:end, ...]
start = end
end += BATCH_SIZE
predict_val, loss_val = network.forward(current_img, current_label)
correct = np.sum(predict_val == current_label)
total_correct += correct
loss += loss_val
total_seen += BATCH_SIZE
log_string('eval mean loss: {}'.format(loss / num_batch))
log_string('eval accuracy: {}'.format(total_correct / total_seen))
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--train-file',
required=True,
help=(
'Training dataset, as per the format from Kaggle: '
'https://www.kaggle.com/c/digit-recognizer/data?select=train.csv'))
parser.add_argument(
'--test-file',
required=True,
help=(
'Testing dataset, as per the format from Kaggle: '
'https://www.kaggle.com/c/digit-recognizer/data?select=test.csv'))
args = parser.parse_args()
train_df = pd.read_csv(args.train_file)
test_df = pd.read_csv(args.test_file)
train_X = train_df.drop(columns=['label']).values
train_y = train_df['label'].values
test_X = test_df.values
model = LeNet()
print(model.classifier)
name = 'lenet_relu'
model.train_on_dataset(train_X, train_y, save_path=name)
test_predictions = model.predict_on_dataset(test_X)
pred_df = pd.DataFrame({
'ImageId': np.arange(1, test_X.shape[0] + 1),
'Label': test_predictions
})
pred_df.to_csv(f'{name}_predictions.csv', index=False)
def __init__(self, img_shape=(160,320,3), model_file="lenet.h5", prev_model=None, batch_size=128, epochs=5):
# net = NvidiaNet()
net = LeNet()
self.nnModel = net.network(img_shape=img_shape)
self.modelLoaded = False
self.modelFile = model_file
self.prevModel = prev_model
self.batchSize = batch_size
self.epochs = epochs
def LeNet_train():
# initializing the network
network = LeNet(BATCH_SIZE)
# load the data
train_imgs, val_imgs, _, train_label, val_label, _ = util.load_data(
MNIST_PATH)
for epoch in range(MAX_EPOCH):
eval_one_epoch(network, val_imgs, val_label)
log_string('------------start train {}/{}----------'.format(
epoch, MAX_EPOCH))
train_one_epoch(network, train_imgs, train_label, epoch)
def inference():
# initializing the network
network = LeNet(BATCH_SIZE)
network.getTheParas(MODEL_FILE)
print(IMAGE_PATH)
image_paths = glob.glob(os.path.join(IMAGE_PATH, '*'))
for image_path in image_paths:
image_data = cv2.imread(image_path, 0)
image_data = image_data[newaxis, :, :, newaxis]
predict_val = network.inference(image_data)
print(image_path, ':', predict_val[0][0])
def aiTest(images, shape):
model = LeNet()
y_test = []
x_test = images
generate_images = []
for image in x_test:
confidence = model.predict(image)[0]
predicted_class = np.argmax(confidence)
y_test.append(predicted_class)
attacker = PixelAttacker((x_test, y_test))
for i in range(len(x_test)):
generate_images.append(attacker.attack(i, model, verbose=False)[10])
return generate_images
def test_image(filename, num_class, weights_path='Default'):
img_string = tf.gfile.FastGFile(filename, 'rb').read()
img_decoded = tf.image.decode_jpeg(img_string, channels=channels)
img_resized = tf.image.resize_images(img_decoded, [image_size, image_size])
img_reshape = tf.reshape(img_resized,
shape=[1, image_size, image_size, channels])
model = LeNet(img_reshape, keep_prob, num_classes, batch_size, image_size,
channels)
score = tf.nn.softmax(model.fc6)
max = tf.argmax(score, 1)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver.restore(sess, "./tmp/checkpoint/model_epoch10.ckpt")
print(sess.run(model.fc6))
prob = sess.run(max)[0]
print("The number is %s" % (class_name[prob]))
def main():
batch_size = 128
epoch = 15
data = DATA()
model = LeNet(data.input_shape, data.num_classes)
hist = model.fit(data.x_train,
data.y_train,
batch_size=batch_size,
epochs=epoch,
validation_split=0.2)
score = model.evaluate(data.x_test, data.y_test, batch_size=batch_size)
print()
print('Test Loss= ', score)
plot_loss(hist)
plt.show()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--model-file',
required=True,
help='File path to a trained PyTorch model')
parser.add_argument(
'--target-digit',
required=True,
type=int,
choices=range(10),
help='The digit that we want to find the \"epitome\" for')
parser.add_argument('--num-samples',
type=int,
default=10,
help=('The number of epitome samples to generate'))
parser.add_argument(
'--use-training-file',
default=None,
help=
('Start constructions with entries from the given training set, '
'rather than from uniform noise. The format of the file should match: '
'https://www.kaggle.com/c/digit-recognizer/data?select=train.csv'))
args = parser.parse_args()
model = LeNet()
model.load_saved_state(args.model_file)
starter_X = None
if args.use_training_file:
train_df = pd.read_csv(args.use_training_file)
starter_df = train_df[train_df['label'] == args.target_digit]
starter_X = starter_df.drop(
columns=['label']).values[:args.num_samples]
opt_inputs = model.optimize_for_digit(
args.target_digit,
num_samples=args.num_samples,
starter_X=starter_X,
)
for i in range(opt_inputs.shape[0]):
img = Image.fromarray(opt_inputs[i], 'L')
img.save(f'digit_{args.target_digit}_{i}.png')
def get_model(self):
if self.exp_type == 'pnml_cifar10':
model = load_pretrained_resnet20_cifar10_model(resnet20())
elif self.exp_type == 'random_labels':
model = WideResNet()
elif self.exp_type == 'out_of_dist_svhn':
model = load_pretrained_resnet20_cifar10_model(resnet20())
elif self.exp_type == 'out_of_dist_noise':
model = load_pretrained_resnet20_cifar10_model(resnet20())
elif self.exp_type == 'pnml_mnist':
model = Net()
elif self.exp_type == 'adversarial':
model = load_pretrained_resnet20_cifar10_model(resnet20())
elif self.exp_type == 'pnml_cifar10_lenet':
model = LeNet() # VGG('VGG16')
else:
raise NameError('No experiment type: %s' % self.exp_type)
return model
def forward_pytorch(weightfile, image):
#net=resnet.resnet18()
#net = resnet.resnet18()
net=LeNet(1,2)
checkpoint = torch.load(weightfile)
net.load_state_dict(checkpoint['weight'])
net.double() # to double
if args.cuda:
net.cuda()
print(net)
net.eval()
image = torch.from_numpy(image.astype(np.float64)) # to double
if args.cuda:
image = Variable(image.cuda())
else:
image = Variable(image)
t0 = time.time()
blobs = net.forward(image)
print(blobs.data.numpy().flatten())
t1 = time.time()
return t1-t0, blobs, net, torch.from_numpy(blobs.data.numpy())
def torch_setup(self):
self.data_train = MNIST('./data/mnist',
train=True,
download=True,
transform=self.compose)
self.data_test = MNIST('./data/mnist',
train=False,
download=True,
transform=self.compose)
self.data_train_loader = DataLoader(self.data_train, batch_size=self.batch_size, shuffle=True, num_workers=4)
self.data_test_loader = DataLoader(self.data_test, batch_size=self.batch_size, num_workers=4)
self.criterion = nn.CrossEntropyLoss()
self.net = LeNet()
self.optimizer = getattr(optim, self.opt)(self.net.parameters(), lr=self.learning_rate)
try:
self.net.load_state_dict(load('model_params.pkl'))
self.loaded = True
print("Loaded")
except Exception as e:
print(e)
self.loaded = False
print("Not loaded")
def main():
global args, rank, world_size, best_prec1, dataset_len
if args.dist == 1:
rank, world_size = dist_init()
else:
rank = 0
world_size = 1
model = LeNet()
model.cuda()
param_copy = [
param.clone().type(torch.cuda.FloatTensor).detach()
for param in model.parameters()
]
for param in param_copy:
param.requires_grad = True
if args.dist == 1:
model = DistModule(model)
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(param_copy,
args.base_lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# optionally resume from a checkpoint
last_iter = -1
# Data loading code
train_dataset = datasets.MNIST(root='./data',
train=True,
transform=transforms.ToTensor(),
download=False)
val_dataset = datasets.MNIST(root='./data',
train=False,
transform=transforms.ToTensor(),
download=False)
dataset_len = len(train_dataset)
args.max_iter = math.ceil(
(dataset_len * args.epoch) / (world_size * args.batch_size))
if args.dist == 1:
train_sampler = DistributedGivenIterationSampler(train_dataset,
args.max_iter,
args.batch_size,
last_iter=last_iter)
val_sampler = DistributedSampler(val_dataset, round_up=False)
else:
train_sampler = DistributedGivenIterationSampler(train_dataset,
args.max_iter,
args.batch_size,
world_size=1,
rank=0,
last_iter=last_iter)
val_sampler = None
# pin_memory if true, will copy the tensor to cuda pinned memory
train_loader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True,
sampler=train_sampler)
val_loader = DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True,
sampler=val_sampler)
train(train_loader, val_loader, model, criterion, optimizer, param_copy)
# REMEMBER TO RIGHTLY SET THE FOLLOWING PATH
TEST_DIR = 'test'
RESULT_DIR = 'results'
# Process test data and create batches in memory
graph = tf.Graph()
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
sess_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=True,
gpu_options=gpu_options)
sess_config.gpu_options.allow_growth = True
sess = tf.Session(config=sess_config)
if TYPE_OF_MODEL == 'LE-NET':
config = Config(TYPE_OF_MODEL)
model = LeNet(config, sess, graph)
elif TYPE_OF_MODEL == 'DEEP-CONVNET':
config = Config(TYPE_OF_MODEL)
model = DeepConv(config, sess, graph)
elif TYPE_OF_MODEL == 'ALEX-NET':
config = Config(TYPE_OF_MODEL)
model = AlexNet(config, sess, graph)
model.restore()
print(TYPE_OF_MODEL + " CNN Model Restored")
test_batches = init_test_data(model.config)
# Get the predictions and write them into a CSV file
with open(RESULT_DIR + '/' + TYPE_OF_MODEL + '.csv', 'wb') as csvfile:
writer = csv.writer(csvfile)
img_grid = vutils.make_grid(data_batch,
nrow=4,
normalize=True,
scale_each=True)
# img_grid = vutils.make_grid(data_batch, nrow=4, normalize=False, scale_each=False)
writer.add_image("input img", img_grid, 0)
writer.close()
# ----------------------------------- 5 add_graph -----------------------------------
# flag = 0
flag = 1
if flag:
writer = SummaryWriter(comment='test_your_comment',
filename_suffix="_test_your_filename_suffix")
# 模型
fake_img = torch.randn(1, 3, 32, 32)
lenet = LeNet(classes=2)
writer.add_graph(lenet, fake_img)
writer.close()
from torchsummary import summary
print(summary(lenet, (3, 32, 32), device="cpu"))
def main():
# Training settings
parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
parser.add_argument('--config_file',
type=str,
default='',
help="config file")
parser.add_argument('--stage',
type=str,
default='',
help="select the pruning stage")
args = parser.parse_args()
config = Config(args)
use_cuda = True
torch.manual_seed(1)
device = torch.device("cuda" if use_cuda else "cpu")
kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
train_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../data',
train=True,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])),
batch_size=64,
shuffle=True,
**kwargs)
test_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../data',
train=False,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])),
batch_size=1000,
shuffle=True,
**kwargs)
model = None
if config.arch == 'lenet_bn':
model = LeNet_BN().to(device)
elif config.arch == 'lenet':
model = LeNet().to(device)
elif config.arch == 'lenet_adv':
model = LeNet_adv(config.width_multiplier).to(device)
torch.cuda.set_device(config.gpu)
model.cuda(config.gpu)
config.model = model
ADMM = None
config.prepare_pruning()
if config.admm:
ADMM = admm.ADMM(config)
criterion = CrossEntropyLossMaybeSmooth(smooth_eps=config.smooth_eps).cuda(
config.gpu)
config.smooth = config.smooth_eps > 0.0
config.mixup = config.alpha > 0.0
config.warmup = (not config.admm) and config.warmup_epochs > 0
optimizer_init_lr = config.warmup_lr if config.warmup else config.lr
if (config.optimizer == 'sgd'):
optimizer = torch.optim.SGD(config.model.parameters(),
optimizer_init_lr,
momentum=0.9,
weight_decay=1e-4)
elif (config.optimizer == 'adam'):
optimizer = torch.optim.Adam(config.model.parameters(),
optimizer_init_lr)
else:
raise Exception("unknown optimizer")
scheduler = None
if config.lr_scheduler == 'cosine':
scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer,
T_max=config.epochs *
len(train_loader),
eta_min=4e-08)
elif config.lr_scheduler == 'default':
pass
else:
raise Exception("unknown lr scheduler")
if config.load_model:
# unlike resume, load model does not care optimizer status or start_epoch
print('==> Loading from {}'.format(config.load_model))
config.model.load_state_dict(
torch.load(config.load_model,
map_location={'cuda:0': 'cuda:{}'.format(config.gpu)}))
test(config, device, test_loader)
global best_acc
if config.resume:
if os.path.isfile(config.resume):
checkpoint = torch.load(config.resume)
config.start_epoch = checkpoint['epoch']
best_acc = checkpoint['best_acc']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
config.resume, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(config.resume))
if config.masked_retrain:
# make sure small weights are pruned and confirm the acc
print("<============masking both weights and gradients for retrain")
admm.masking(config)
print("<============testing sparsity before retrain")
admm.test_sparsity(config)
test(config, device, test_loader)
if config.masked_progressive:
admm.zero_masking(config)
for epoch in range(0, config.epochs + 1):
train(config, ADMM, device, train_loader, criterion, optimizer,
scheduler, epoch)
test(config, device, test_loader)
save_checkpoint(
config, {
'epoch': epoch + 1,
'arch': config.arch,
'state_dict': config.model.state_dict(),
'best_acc': best_acc,
'optimizer': optimizer.state_dict()
})
print('overall best_acc is {}'.format(best_acc))
if (config.save_model and config.admm):
print('saving model {}'.format(config.save_model))
torch.save(config.model.state_dict(), config.save_model)
# prepare the train & test dataset and labels
# expand 3D dataset to 4D dataset
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = np.reshape(x_train,
(x_train.shape[0], x_train.shape[1], x_train.shape[2], 1))
x_test = np.reshape(x_test,
(x_test.shape[0], x_test.shape[1], x_test.shape[2], 1))
# convert the labels form integers to vectors
lb = LabelBinarizer()
y_train = lb.fit_transform(y_train)
y_test = lb.fit_transform(y_test)
# define the model
lenet = LeNet()
model = lenet.net
# optimizer
sgd = SGD(momentum=0.9, nesterov=True)
print("[INFO] start training...")
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=["accuracy"])
H = model.fit(x_train,
y_train,
validation_data=(x_test, y_test),
epochs=100,
batch_size=128)
torchvision.datasets.ImageFolder(args.train, transform = data_transforms),
batch_size=args.batch_size, shuffle=True,
num_workers=args.worker
)
val_loader = torch.utils.data.DataLoader(
torchvision.datasets.ImageFolder(args.val, transform = data_transforms),
batch_size=args.batch_size, shuffle=True,
num_workers=args.worker
)
return train_loader, val_loader
train_loader, val_loader = get_dataloader()
classes = 10
net = LeNet(classes)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.9)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# model to gpu
net.to(device)
print("Start Training...")
os.makedirs('./expr', exist_ok=True)
for epoch in range(1000):
net.train()
running_loss = 0.0
for i, data in enumerate(train_loader):
inputs, labels = data
inputs, labels = inputs.to(device, dtype=torch.float), labels.to(device)
'resource_info')),
'Filename containing cluster information')
tf.app.flags.DEFINE_integer('max_steps', 1000000,
"""Number of iterations to run for each workers.""")
tf.app.flags.DEFINE_integer('log_frequency', 50,
"""How many steps between two logs.""")
tf.app.flags.DEFINE_integer('batch_size', 32,
"""Batch size""")
tf.app.flags.DEFINE_boolean('sync', True, '')
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
# Build single-GPU LeNet model
single_gpu_graph = tf.Graph()
with single_gpu_graph.as_default():
model = LeNet()
parallax_config = parallax.Config()
ckpt_config = parallax.CheckPointConfig(ckpt_dir='ckpt',
save_ckpt_steps=FLAGS.log_frequency)
parallax_config.ckpt_config = ckpt_config
sess, num_workers, worker_id, num_replicas_per_worker = parallax.parallel_run(
single_gpu_graph,
FLAGS.resource_info_file,
sync=FLAGS.sync,
parallax_config=parallax_config)
start = time.time()
for i in range(FLAGS.max_steps):
batch = mnist.train.next_batch(FLAGS.batch_size, shuffle=False)
#!/usr/bin/env python3
import torch as th
import torchvision as tv
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable as V
from torchvision import transforms
from lenet import LeNet
teacher = LeNet()
teacher = teacher.cuda()
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
trainset = tv.datasets.CIFAR10(root='./data',
train=True,
download=True,
transform=transform_train)
"Allow device soft device placement")
tf.flags.DEFINE_boolean("log_device_placement", False,
"Log placement of ops on devices")
FLAGS = tf.flags.FLAGS
(x_train_val, y_train_val), (
x_test, y_test) = load_data() # training data: 50000, test data: 10000
x_train, y_train, x_test, y_test, x_val, y_val = dh.shuffle_data(
x_train_val, y_train_val, x_test, y_test, FLAGS.num_classes)
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
sess = tf.Session(config=session_conf)
with sess.as_default():
lenet = LeNet(FLAGS)
# Define Training procedure
# * hint learning rate decay를 위한 operation을 통해 감쇠된 learning rate를 optimizer에 적용
learning_rate = FLAGS.starter_learning_rate # Learning rate decay가 적용되지 않는다면 Starter learning rate값이 그대로 전달됨
global_step = tf.Variable(0, name="global_step",
trainable=False) # iteration 수
if FLAGS.learning_rate_decay_rate != 1:
learning_rate = learning_rate * FLAGS.learning_rate_decay_rate**(
global_step / FLAGS.learning_rate_decay_step)
if FLAGS.optimizer == 'adam':
optimizer = tf.train.AdamOptimizer(
learning_rate=learning_rate) #Optimizer
elif FLAGS.optimizer == 'adagrad':
optimizer = tf.train.AdagradOptimizer(
learning_rate=learning_rate)
def main():
parser = argparse.ArgumentParser()
mode_group = parser.add_mutually_exclusive_group(required=True)
mode_group.add_argument("--train",
action="store_true",
help="To train the network.")
mode_group.add_argument("--test",
action="store_true",
help="To test the network.")
parser.add_argument("--epochs",
default=10,
type=int,
help="Desired number of epochs.")
parser.add_argument("--dropout",
action="store_true",
help="Whether to use dropout or not.")
parser.add_argument("--uncertainty",
action="store_true",
help="Use uncertainty or not.")
parser.add_argument("--dataset",
action="store_true",
help="The dataset to use.")
parser.add_argument("--outsample",
action="store_true",
help="Use out of sample test image")
uncertainty_type_group = parser.add_mutually_exclusive_group()
uncertainty_type_group.add_argument(
"--mse",
action="store_true",
help=
"Set this argument when using uncertainty. Sets loss function to Expected Mean Square Error."
)
uncertainty_type_group.add_argument(
"--digamma",
action="store_true",
help=
"Set this argument when using uncertainty. Sets loss function to Expected Cross Entropy."
)
uncertainty_type_group.add_argument(
"--log",
action="store_true",
help=
"Set this argument when using uncertainty. Sets loss function to Negative Log of the Expected Likelihood."
)
dataset_type_group = parser.add_mutually_exclusive_group()
dataset_type_group.add_argument(
"--mnist",
action="store_true",
help="Set this argument when using MNIST dataset")
dataset_type_group.add_argument(
"--emnist",
action="store_true",
help="Set this argument when using EMNIST dataset")
dataset_type_group.add_argument(
"--CIFAR",
action="store_true",
help="Set this argument when using CIFAR dataset")
dataset_type_group.add_argument(
"--fmnist",
action="store_true",
help="Set this argument when using FMNIST dataset")
args = parser.parse_args()
if args.dataset:
if args.mnist:
from mnist import dataloaders, label_list
elif args.CIFAR:
from CIFAR import dataloaders, label_list
elif args.fmnist:
from fashionMNIST import dataloaders, label_list
if args.train:
num_epochs = args.epochs
use_uncertainty = args.uncertainty
num_classes = 10
model = LeNet(dropout=args.dropout)
if use_uncertainty:
if args.digamma:
criterion = edl_digamma_loss
elif args.log:
criterion = edl_log_loss
elif args.mse:
criterion = edl_mse_loss
else:
parser.error(
"--uncertainty requires --mse, --log or --digamma.")
else:
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=1e-3, weight_decay=0.005)
exp_lr_scheduler = optim.lr_scheduler.StepLR(optimizer,
step_size=7,
gamma=0.1)
device = get_device()
model = model.to(device)
model, metrics = train_model(model,
dataloaders,
num_classes,
criterion,
optimizer,
scheduler=exp_lr_scheduler,
num_epochs=num_epochs,
device=device,
uncertainty=use_uncertainty)
state = {
"epoch": num_epochs,
"model_state_dict": model.state_dict(),
"optimizer_state_dict": optimizer.state_dict(),
}
if use_uncertainty:
if args.digamma:
torch.save(state, "./results/model_uncertainty_digamma.pt")
print("Saved: ./results/model_uncertainty_digamma.pt")
if args.log:
torch.save(state, "./results/model_uncertainty_log.pt")
print("Saved: ./results/model_uncertainty_log.pt")
if args.mse:
torch.save(state, "./results/model_uncertainty_mse.pt")
print("Saved: ./results/model_uncertainty_mse.pt")
else:
torch.save(state, "./results/model.pt")
print("Saved: ./results/model.pt")
elif args.test:
use_uncertainty = args.uncertainty
device = get_device()
model = LeNet()
model = model.to(device)
optimizer = optim.Adam(model.parameters())
if use_uncertainty:
if args.digamma:
checkpoint = torch.load(
"./results/model_uncertainty_digamma.pt")
if args.log:
checkpoint = torch.load("./results/model_uncertainty_log.pt")
if args.mse:
checkpoint = torch.load("./results/model_uncertainty_mse.pt")
else:
checkpoint = torch.load("./results/model.pt")
filename = "./results/rotate.jpg"
model.load_state_dict(checkpoint["model_state_dict"])
optimizer.load_state_dict(checkpoint["optimizer_state_dict"])
model.eval()
if args.outsample:
img = Image.open("./data/arka.jpg").convert('L').resize((28, 28))
img = TF.to_tensor(img)
img.unsqueeze_(0)
else:
a = iter(dataloaders['test'])
img, label = next(a)
rotating_image_classification(model,
img,
filename,
label_list,
uncertainty=use_uncertainty)
img = transforms.ToPILImage()(img[0][0])
test_single_image(model, img, label_list, uncertainty=use_uncertainty)
tf.flags.DEFINE_boolean("log_device_placement", False,
"Log placement of ops on devices")
FLAGS = tf.flags.FLAGS
(x_train_val,
y_train_val), (x_test,
y_test) = load_data() # training data: 50000, test data: 10000
x_train, y_train, x_test, y_test, x_val, y_val = dh.shuffle_data(
x_train_val, y_train_val, x_test, y_test, FLAGS.num_classes)
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
sess = tf.Session(config=session_conf)
with sess.as_default():
lenet = LeNet(
FLAGS) #LeNet 클래스의 인스턴스 생성 후 Hyperparameter가 정의돼 있는 FLAGS로 초기화
# Define Training procedure
global_step = tf.Variable(0, name="global_step",
trainable=False) # iteration 수
# * hint learning rate decay를 위한 operation을 통해 감쇠된 learning rate를 optimizer에 적용
# decayed_learning_rate = learning_rate * decay_rate ^ (global_step / decay_steps)
# decayed_learning_rate = FLAGS.lr * FLAGS.lr_decay ** (global_step / FLAGS.lr_decay_steps)
# optimizer = tf.train.AdamOptimizer(learning_rate=decayed_learning_rate)
optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.lr) #Optimizer
grads_and_vars = optimizer.compute_gradients(lenet.loss) # gradient 계산
train_op = optimizer.apply_gradients(
grads_and_vars, global_step=global_step) # back-propagation
# Output directory for models and summaries
#!/usr/bin/env python3
import torch as th
import torchvision as tv
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable as V
from torchvision import transforms
from lenet import LeNet
from sobolev import SobolevLoss
USE_SOBOLEV = False
student = LeNet()
teacher = LeNet()
teacher.load_state_dict(th.load('teacher.pth'))
student = student.cuda()
teacher = teacher.cuda()
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
def train_lenet(split_dir):
"""训练lenet"""
set_seed() # 设置随机种子
rmb_label = {"1": 0, "100": 1}
# 参数设置
MAX_EPOCH = 10
BATCH_SIZE = 16
LR = 0.01
log_interval = 10
val_interval = 1
"""Step 1: 数据读取"""
train_dir = os.path.join(split_dir, "train")
valid_dir = os.path.join(split_dir, "valid")
test_dir = os.path.join(split_dir, "test")
norm_mean = [0.485, 0.456, 0.406]
norm_std = [0.229, 0.224, 0.225]
# 对训练数据进行变换,添加RandomCrop进行数据增强
train_transform = transforms.Compose([
transforms.Resize((32, 32)),
transforms.RandomCrop(32, padding=4),
transforms.ToTensor(),
transforms.Normalize(norm_mean, norm_std),
])
# 对验证数据进行变换
valid_transform = transforms.Compose([
transforms.Resize((32, 32)),
transforms.ToTensor(),
transforms.Normalize(norm_mean, norm_std),
])
# 构建RMBDataset实例
train_data = RMBDataset(data_dir=train_dir, transform=train_transform)
valid_data = RMBDataset(data_dir=valid_dir, transform=valid_transform)
# 构建DataLoader
train_loader = DataLoader(dataset=train_data,
batch_size=BATCH_SIZE,
shuffle=True)
valid_loader = DataLoader(dataset=valid_data, batch_size=BATCH_SIZE)
"""Step 2: 模型"""
net = LeNet(classes=2)
net.initialize_weights()
"""Step 3: 损失函数"""
criterion = nn.CrossEntropyLoss()
"""Step 4: 优化器"""
optimizer = optim.SGD(net.parameters(), lr=LR, momentum=0.9)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=10,
gamma=0.1)
"""Step 5: 训练"""
train_curve = []
valid_curve = []
figure_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'result')
for epoch in range(MAX_EPOCH):
loss_mean = 0.
correct = 0.
total = 0.
net.train()
for i, data in enumerate(train_loader):
# forward
inputs, labels = data
outputs = net(inputs)
# backward
optimizer.zero_grad()
loss = criterion(outputs, labels)
loss.backward()
# update weights
optimizer.step()
# 统计分类情况
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).squeeze().sum().numpy()
# 打印训练信息
loss_mean += loss.item()
train_curve.append(loss.item())
if (i + 1) % log_interval == 0:
loss_mean = loss_mean / log_interval
print(
"Training:Epoch[{:0>3}/{:0>3}] Iteration[{:0>3}/{:0>3}] Loss: {:.4f} Acc:{:.2%}"
.format(epoch, MAX_EPOCH, i + 1, len(train_loader),
loss_mean, correct / total))
loss_mean = 0.
scheduler.step() # 更新学习率
# validate the model
if (epoch + 1) % val_interval == 0:
correct_val = 0.
total_val = 0.
loss_val = 0.
net.eval()
with torch.no_grad():
for j, data in enumerate(valid_loader):
inputs, labels = data
outputs = net(inputs)
loss = criterion(outputs, labels)
_, predicted = torch.max(outputs.data, 1)
total_val += labels.size(0)
correct_val += (
predicted == labels).squeeze().sum().numpy()
loss_val += loss.item()
valid_curve.append(loss_val / valid_loader.__len__())
print(
"Valid:\t Epoch[{:0>3}/{:0>3}] Iteration[{:0>3}/{:0>3}] Loss: {:.4f} Acc:{:.2%}"
.format(epoch, MAX_EPOCH, j + 1, len(valid_loader),
loss_val, correct_val / total_val))
train_x = range(len(train_curve))
train_y = train_curve
train_iters = len(train_loader)
valid_x = np.arange(
1,
len(valid_curve) + 1
) * train_iters * val_interval # 由于valid中记录的是epochloss，需要对记录点进行转换到iterations
valid_y = valid_curve
plt.plot(train_x, train_y, label='Train')
plt.plot(valid_x, valid_y, label='Valid')
plt.legend(loc='upper right')
plt.ylabel('loss value')
plt.xlabel('Iteration')
figure_path = os.path.join(figure_dir, '0201.png')
plt.savefig(figure_path)
plt.close()
batch = get_batch(training_loader)
_, _, height, width = batch['image'].shape
shape = height, width
shape = net_input_size, net_input_size
# https://discuss.pytorch.org/t/how-can-l-load-my-best-model-as-a-feature-extractor-evaluator/17254/6
activation = {}
def get_activation(name):
def hook(model, input, output):
activation[name] = output.detach()
return hook
# Actual network
net = LeNet(shape)
net.fc2.register_forward_hook(get_activation('fc2'))
net.conv2.register_forward_hook(get_activation('conv2'))
# Log architecture
dummy_input = torch.zeros(1, 1, net_input_size, net_input_size)
writer.add_graph(
net,
input_to_model=dummy_input,
# verbose=True,
)
# Loss
criterion = nn.CrossEntropyLoss()
# Optimizer
cuda = not args.disable_cuda and torch.cuda.is_available()
trainset, testset, classes, shape = get_mnist(input_dimensions=2)
w, h = shape
trainloader = torch.utils.data.DataLoader(trainset,
batch_size=args.batch_size,
shuffle=True,
pin_memory=cuda)
testloader = torch.utils.data.DataLoader(testset,
batch_size=args.batch_size,
shuffle=False,
pin_memory=cuda)
model = LeNet(k=len(classes), args=args)
if cuda:
model.cuda()
for epoch in range(1, args.epochs + 1):
model.train()
for i, (x_batch, y_batch) in enumerate(trainloader):
if cuda:
x_batch, y_batch = x_batch.cuda(), y_batch.cuda()
model.optimizer.zero_grad()
# forward + backward + optimize
output = model(x_batch)