def __init__(self):
# initialize dataset with PCA
n_samples, tempotrons = 5, []
dataset = MNIST(n_components=100)
# training phase
for training_digit in list(range(10)):
print('training tempotron to recognize {}...'.format(training_digit))
tempotrons.append(DigitTempotron(dataset, training_digit)) # auto-trains
X_train, y_train = np.zeros((n_samples*10, 10)), np.zeros((n_samples*10, 1))
# classification phase
with open('X_train.txt', 'r+') as f:
if RW == 'w':
for i in range(n_samples*10):
images = dataset.sample(n_samples, int(i/n_samples), int(i/n_samples)) # five sample vectors, each one of 0->9
y_train[i] = int(i/n_samples)
print('')
for t in range(10):
truth, t_max, X_train[i][t] = tempotrons[t].classify(images[i % n_samples])
f.write(str(X_train[i][t])+'\n')
print('tempotron {} classified digit {} v{} as:\t{}\t{}'\
.format(t, int(i/n_samples), i%n_samples, truth, X_train[i][t]))
if RW == 'r':
vals = [float(y) for y in list(filter(lambda x: x != '', [line.strip() for line in f.readlines()]))]
for i in range(len(vals)): # n_samples*10^2
X_train[int(i/10)][i%10] = vals[i]
for j in range(n_samples*10):
y_train[j] = int(j/n_samples)
self.tempotrons = tempotrons
self.X_train, self.y_train = X_train, y_train
def main():
""" Main function
Here, you should instantiate
1) Dataset objects for training and test datasets
2) DataLoaders for training and testing
3) model
4) optimizer: SGD with initial learning rate 0.01 and momentum 0.9
5) cost function: use torch.nn.CrossEntropyLoss
"""
dir = "./data"
wandb.init(project="mnist_lenet", entity='hohyun')
transform = {
'train':
transforms.Compose([
transforms.RandomRotation(5),
transforms.RandomHorizontalFlip(),
transforms.RandomResizedCrop(32,
scale=(0.96, 1.0),
ratio=(0.95, 1.05)),
transforms.ToTensor()
]),
'test':
transforms.Compose(
[transforms.Resize([32, 32]),
transforms.ToTensor()])
}
batch_size = 32
train_set = MNIST(data_dir=dir, mode='train', transform=transform['train'])
trn_loader = DataLoader(train_set,
batch_size=batch_size,
shuffle=True,
num_workers=0)
test_set = MNIST(data_dir=dir, mode='test', transform=transform['test'])
tst_loader = DataLoader(test_set,
batch_size=batch_size,
shuffle=False,
num_workers=0)
device = "cuda" if torch.cuda.is_available() else "cpu"
epochs = 10
criterion = nn.CrossEntropyLoss().to(device)
# model = LeNet5().to(device)
model = CustomMLP().to(device)
optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
for i in range(epochs):
print(f"------{i+1} epoch in progress------")
train(model, trn_loader, device, criterion, optimizer)
test(model, tst_loader, device, criterion)
def main():
""" Main function
Here, you should instantiate
1) Dataset objects for training and test datasets
2) DataLoaders for training and testing
3) model
4) optimizer: SGD with initial learning rate 0.01 and momentum 0.9
5) cost function: use torch.nn.CrossEntropyLoss
"""
# write your codes here
USE_CUDA = torch.cuda.is_available()
DEVICE = torch.device("cuda" if USE_CUDA else "cpu")
EPOCHS = 10
BATCH_SIZE = 64
model = LeNet5().to(DEVICE)
#model = CustomMLP().to(DEVICE)
#optimizer = optim.SGD(model.parameters(), lr = 0.01, momentum = 0.5)
optimizer = optim.SGD(model.parameters(),
lr=0.01,
momentum=0.5,
weight_decay=0.001)
d = "C:/Users/inolab/Desktop/DNN/CNN_homework/data"
transform_train = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))])
transform_test = transforms.Compose([transforms.ToTensor()])
train_set = MNIST(data_dir=d, folder='train', transform=transform_train)
train_loader = DataLoader(train_set,
batch_size=BATCH_SIZE,
shuffle=False,
num_workers=0)
test_set = MNIST(data_dir=d, folder='test', transform=transform_test)
tst_loader = DataLoader(test_set,
batch_size=BATCH_SIZE,
shuffle=False,
num_workers=0)
for epoch in range(1, EPOCHS + 1):
trn_loss, tr_acc = train(model, train_loader, DEVICE, optimizer, epoch)
tst_loss, acc = test(model, tst_loader, DEVICE)
print('[{}] Test Loss : {:.4f}, Test Acc: {:.2f}%'.format(
epoch, tst_loss, acc))
def get_data_loader(task, batch_size=1, split='train'):
# NOTE: batch size here is # instances PER CLASS
if task.dataset == 'mnist':
normalize = transforms.Normalize(mean=[0.13066, 0.13066, 0.13066],
std=[0.30131, 0.30131, 0.30131])
dset = MNIST(task,
transform=transforms.Compose(
[transforms.ToTensor(), normalize]),
split=split)
else:
normalize = transforms.Normalize(mean=[0.92206, 0.92206, 0.92206],
std=[0.08426, 0.08426, 0.08426])
dset = Omniglot(task,
transform=transforms.Compose(
[transforms.ToTensor(), normalize]),
split=split)
sampler = ClassBalancedSampler(
task.num_cl,
task.num_inst,
batch_cutoff=(None if split != 'train' else batch_size))
loader = DataLoader(dset,
batch_size=batch_size * task.num_cl,
sampler=sampler,
num_workers=0,
pin_memory=True)
return loader
def train(load_model, model_path):
model = MLP_GAN()
if load_model:
print('Loading model...')
model.load(model_path)
print('...Done.')
ds = MNIST()
teacher = UnsupervisedTeacher(model, ds)
if vis.check_connection():
teacher.train(epoch_callback=epoch_callback(model),
batch_callback=batch_callback(model))
else:
teacher.train(epoch_callback=epoch_callback(model))
def main(params):
ds = MNIST()
x_train, y_train = ds.get_train()
x_val, y_val = ds.get_val()
print(x_train.shape, np.bincount(y_train), x_val.shape, np.bincount(y_val))
model_holder = SampleCNN()
model = model_holder.build_model(ds.get_input_shape(), ds.get_nb_classes())
loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
metrics = ['sparse_categorical_accuracy']
model.compile(tf.keras.optimizers.Adam(1e-4), loss_fn, metrics)
m_path = os.path.join(params.save_dir, model_holder.get_name())
util.mk_parent_dir(m_path)
callbacks = [
tf.keras.callbacks.ModelCheckpoint(m_path +
'_{epoch:03d}-{val_loss:.2f}.hdf5'),
tf.keras.callbacks.CSVLogger(
os.path.join(params.save_dir, 'metrics.csv'))
]
model.fit(x_train,
y_train,
epochs=params.epoch,
validation_data=(x_val, y_val),
batch_size=params.batch_size,
callbacks=callbacks)
import numpy as np
from keras.models import Sequential # Keras Model
from keras.layers import Dense, Dropout, Activation, Flatten # Keras Layers
from keras.layers import Convolution2D, MaxPooling2D # Keras CNN Layers
from keras.utils import np_utils
from matplotlib import pyplot as plt
from keras.datasets import mnist
from tempo_layer import DigitTempotronLayer
from dataset import MNIST
# With 50 training images and 50 testing images
# x_train should be 50*10*10 matrix, x_test should be 50*10*10 matrix
# y_train should be 50*1 vector, y_test should be 50*1 vector
dtl = DigitTempotronLayer()
dataset = MNIST(n_components=100, reshape=False)
np.random.seed(7) # for reproducibility
# Training data
train_samples = []
for digit in range(10): # 0->9
for ten_by_ten_matrix in dataset.sample(5, digit, digit): # 5 x 'digit'
train_samples.append(ten_by_ten_matrix)
train_samples = np.asarray(train_samples).reshape(50, 100)
X_train = train_samples.astype('float32')
y_train = dtl.get_layer_output()[1]
Y_train = np_utils.to_categorical(y_train) # 50*10 one hot matrix (encoded outputs)
# Testing data
i = 0
test_samples = []
#set parameters
#whether gpu is available
cuda = torch.cuda.is_available()
device = torch.device("cuda" if cuda else "cpu")
#create model
model = model_cnn()
if (cuda):
model = model.to(device)
#load model
model.load_state_dict(torch.load('params.pkl'))
model.eval()
#create validation dataset
testing_dataset = MNIST(test_data, test_label)
#create train loader
kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {}
test_loader = torch.utils.data.DataLoader(testing_dataset,
batch_size=1024,
shuffle=False,
**kwargs)
#create model loss function
loss_func = torch.nn.CrossEntropyLoss()
current = 0
prediction = []
for index, data in enumerate(test_loader):
img, label = data
from keras.layers import Dense, Dropout, Activation, Flatten # Keras Layers
from keras.layers import Convolution2D, MaxPooling2D # Keras CNN Layers
from keras.utils import np_utils
from matplotlib import pyplot as plt
from keras.datasets import mnist
from tempo_layer import DigitTempotronLayer
from dataset import MNIST
np.random.seed(7) # for reproducibility
# With 50 training images and 50 testing images
# x_train should be 50*10*10 matrix, x_test should be 50*10*10 matrix
# y_train should be 50*1 vector, y_test should be 50*1 vector
dtl = DigitTempotronLayer()
dataset = MNIST(n_components=10)
samples = []
for digit in range(10): # 0->9
for vector in dataset.sample(5, digit, digit): # 5 x 'digit'
samples.append(vector)
samples = np.asarray(samples)
# Preprocess Input Matrices
X_train, X_test = [samples.astype('float32') for i in range(2)]
y_train, y_test = [dtl.get_layer_output()[1] for i in range(2)]
# Y_train should be 50*10 one hot matrix (encoded outputs)
# Y_test should be 50*10 one hot matrix (encoded outputs)
Y_train = np_utils.to_categorical(y_train)
Y_test = np_utils.to_categorical(y_test)
# if i % 5000 == 0:
# avg.description()
# model.saver_save(sess)
# fine tune the model
meta_test_Dtrain = MiniImageNet('test')
meta_test_Dtest = MiniImageNet('test_val')
if args.dataset=='imgnet':
meta_test_Dtrain = MiniImageNet('test')
meta_test_Dtest = MiniImageNet('test_val')
if args.dataset=='cifar':
meta_test_Dtrain = Cifar('cifar10', 'train')
meta_test_Dtest = Cifar('cifar10', 'test')
if args.dataset=='mnist':
meta_test_Dtrain = MNIST('MNIST', 'train')
meta_test_Dtest = MNIST('MNIST', 'test')
# random_class_nway = np.random.choice(20,
# size=args.nway, replace=False)
# random_class_nway = [2,6,3,14,11]
random_class_nway = [1,2,3,4,5]
Dtrain = SubDataSet(random_class_nway,
(meta_test_Dtrain.x, meta_test_Dtrain.y), k=500)
Dtest = SubDataSet(random_class_nway,
(meta_test_Dtest.x, meta_test_Dtest.y))
for i in range(1, 5001):
lr = 1e-4
batch_x, batch_y = Dtrain.next_batch(32)
fd = {x_ph: batch_x, y_ph:batch_y, lr_ph: lr}
p1, p2, _ = sess.run([acc2, loss2, train_step2], fd)
from __future__ import division
import torch
from torch import nn
from torch.optim import Adam
import numpy as np
import matplotlib.pyplot as plt
import itertools
from dataset import MNIST
from models import Generator, Discriminator
max_epoch = 150
batch_size = 128
z_dim = 100
train_loader, test_loader = MNIST(128)
G = Generator(z_dim, 28 * 28).cuda()
D = Discriminator(28 * 28).cuda()
G_opt = Adam(G.parameters(), lr=0.0002)
D_opt = Adam(D.parameters(), lr=0.0002)
loss_func = nn.BCELoss().cuda()
zs_fixed = torch.randn((10 * 10, 100))
def plot_results():
zs = torch.randn((10 * 10, 100)).cuda()
gen_imgs = G(zs)
fig, ax = plt.subplots(10, 10)
for i, j in itertools.product(range(10), range(10)):
from tqdm import tqdm
import numpy as np
import tensorboardX
from sklearn.metrics import f1_score
N_EPOCHS = 100
N_CLASSES = 10
BATCH_SIZE = 1000
MAX_LR = 1e-4
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
if __name__ == '__main__':
# load datasets
trainset = MNIST("../../Data/MNIST/test/")
testset = MNIST("../../Data/MNIST/test/")
trainloader = DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=True)
testloader = DataLoader(testset, batch_size=BATCH_SIZE, shuffle=False)
# init Model
model = Net()
model = model.to(device)
# init optimizer, lr_sched & loss
optimizer = Adam(model.parameters(), lr=MAX_LR)
scheduler = OneCycleLR(optimizer, max_lr=MAX_LR, epochs=100, steps_per_epoch=len(trainloader))
nllloss = nn.NLLLoss()
log = tensorboardX.SummaryWriter("MNIST_SVM_3_3")
import tensorflow as tf
import tensorflow_addons as tfa
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping
from config import TrainingConfig, BayesOptConfig, BoundConfig
from dataset import MNIST
from cnn import CnnSearchSpace
training_config = TrainingConfig(
dataset=MNIST(binary=True),
epochs=25,
batch_size=128,
optimizer=lambda: tfa.optimizers.AdamW(learning_rate=0.001,
weight_decay=0.0001),
callbacks=lambda: [EarlyStopping(patience=10, min_delta=0.005)],
)
search_config = BayesOptConfig(search_space=CnnSearchSpace(),
starting_points=10,
checkpoint_dir="artifacts/cnn_mnist")
bound_config = BoundConfig(error_bound=0.01,
peak_mem_bound=2000,
model_size_bound=2000,
mac_bound=1000000)
import time
import executor.psgd as PSGD
import nn
from codec.quantization import QuantizedClient, QuantizedParaServer
from dataset import MNIST
from dataset.transforms import ImageCls, Shuffle
if __name__ == '__main__':
model = nn.model.Model.load('MNISTDNN.model')
job = PSGD.ParallelSGD(model,
data=MNIST(),
transform=Shuffle().add(ImageCls()))
for i in range(1, 11):
try:
nodes = PSGD.parse_worker(worker_cnt=i, ps=False)
job.parallel(nodes,
codec=QuantizedClient,
epoch=80,
op_type=nn.optimizer.GradientAveragingOptimizer,
ps_codec=QuantizedParaServer,
gd_type=nn.gradient_descent.SGDOptimizer,
gd_params=(0.005, ),
mission_title="FP[{}nodes]".format(i),
ssgd_timeout_limit=1000)
except ConnectionAbortedError:
print("Worker exited without reports.")
time.sleep(10)
lr = 5e-3 epc = 200 print " " print "pca_retain =", pca_retain print "hid_layer_sizes =", hid_layer_sizes print "batchsize =", batchsize print "momentum =", momentum print "beta_1 = %f, beta2 = %f" % (beta_1, beta_2) print "lr = %f, epc = %d" % (lr, epc) ############# # LOAD DATA # ############# MNIST_data = MNIST() train_x, train_y = MNIST_data.get_train_set(include_valid=True) test_x, test_y = MNIST_data.get_test_set() print "\n... pre-processing" preprocess_model = SubtractMeanAndNormalizeH(train_x.shape[1]) map_fun = theano.function([preprocess_model.varin], preprocess_model.output()) train_x = map_fun(train_x) test_x = map_fun(test_x) train_x = theano.shared(value=train_x, name='train_x', borrow=True) train_y = theano.shared(value=train_y, name='train_y', borrow=True) test_x = theano.shared(value=test_x, name='test_x', borrow=True) test_y = theano.shared(value=test_y, name='test_y', borrow=True) print "Done."
def main(args):
# first, we define some pre-processing
data_transforms = transforms.Compose([
# extra augmentations
# transforms.ColorJitter(brightness=0.3),
# necessary transformations
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])
# build our data loaders
train_dataset = MNIST('../data', train=True, download=True, transform=data_transforms)
test_dataset = MNIST('../data', train=False, transform=data_transforms)
train_dataloader = DataLoader(train_dataset, batch_size=args.batch_size)
test_dataloader = DataLoader(test_dataset, batch_size=args.batch_size)
# create the model
model = Net()
# build an optimizer for optimizing the parameters of our model
optimizer = Adadelta(model.parameters(), lr=args.lr)
# if we want to use cuda, we have to copy all parameters to the GPU
model.to(args.device)
# build object that handles updating routine
updater = MNISTUpdater(
iterators={'images': train_dataloader},
networks={'net': model},
optimizers={'main': optimizer},
device=args.device,
copy_to_device=True,
)
# build the trainer
trainer = Trainer(
updater,
stop_trigger=get_trigger((args.epochs, 'epoch'))
)
# prepare logging
logger = TensorboardLogger(
args.log_dir,
args,
{},
Path(__file__).resolve().parent,
trigger=get_trigger((100, 'iteration'))
)
# make sure we are evaluating
trainer.extend(Evaluator(
test_dataloader,
logger,
MNISTEvaluator(model),
args.device,
))
# make sure we are saving the trained models to disk, including the optimizer. This allows us to resume training.
snapshotter = Snapshotter(
{
'network': model,
'optimizer': optimizer
},
args.log_dir
)
trainer.extend(snapshotter)
# add learning rate scheduling, in this case Cosine Annealing
schedulers = {
"encoder": CosineAnnealingLR(optimizer, trainer.num_epochs * trainer.iterations_per_epoch, eta_min=1e-8)
}
lr_scheduler = LRScheduler(schedulers, trigger=get_trigger((1, 'iteration')))
trainer.extend(lr_scheduler)
trainer.extend(logger)
trainer.train()
from tensorflow import keras
from dataset import CIFAR, MNIST
from dataset.transforms import ImageCls
if __name__ == '__main__':
x, y, x_t, y_t = ImageCls()(*MNIST().load())
model = keras.Sequential()
model.add(keras.layers.Reshape(target_shape=(28, 28, 1)))
model.add(
keras.layers.Conv2D(filters=64, kernel_size=[5, 5], activation='relu'))
model.add(
keras.layers.Conv2D(filters=64, kernel_size=[5, 5], activation='relu'))
# model.add(keras.layers.MaxPool2D())
model.add(
keras.layers.Conv2D(filters=64, kernel_size=[5, 5], activation='relu'))
model.add(
keras.layers.Conv2D(filters=64, kernel_size=[5, 5], activation='relu'))
# model.add(keras.layers.MaxPool2D())
model.add(keras.layers.Flatten())
model.add(keras.layers.Dense(units=128, activation='tanh'))
model.add(keras.layers.Dense(units=10, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=keras.optimizers.Adam(),
metrics=['accuracy'])
model.fit(x, y, epochs=40, batch_size=100)
nb_filters = 64
layers = [
Conv2D(nb_filters, (5, 5),
strides=(2, 2),
padding="same",
input_shape=(self.image_size, self.image_size,
self.num_channels)),
Activation('relu'),
Conv2D(nb_filters, (3, 3), strides=(2, 2), padding="valid"),
Activation('relu'),
Conv2D(nb_filters, (3, 3), strides=(1, 1), padding="valid"),
Activation('relu'),
Flatten(),
Dense(32),
Activation('relu'),
Dropout(.5),
Dense(self.num_labels)
]
for layer in layers:
model.add(layer)
if restore != None:
model.load_weights(restore)
self.model = model
if __name__ == '__main__':
model = CNN(MNIST())
import numpy as np
from keras.models import Sequential # Keras Model
from keras.layers import Dense, Dropout, Activation, Flatten # Keras Layers
from keras.layers import Convolution2D, MaxPooling2D # Keras CNN Layers
from keras.utils import np_utils
from matplotlib import pyplot as plt
from keras.datasets import mnist
from dataset import MNIST
from tempo_layer import DigitTempotronLayer
# With 50 training images and 50 testing images
# x_train should be 50*10*10 matrix, x_test should be 50*10*10 matrix
# y_train should be 50*1 vector, y_test should be 50*1 vector
dtl = DigitTempotronLayer()
dataset = MNIST(n_components=100, reshape=False)
np.random.seed(7) # for reproducibility
max_voltage = 64
# Training data
x_train, y_train = dtl.get_layer_output()
X_train = x_train.astype('float32') / max_voltage # normalize
Y_train = np_utils.to_categorical(y_train) # 50*10 one hot matrix (encoded outputs)
# Testing data
y_test = np.array(y_train, copy=True)
x_test = np.array(x_train, copy=True)
new_x = []
for digit in range(10): # 0->9
for vector in dataset.new_sample(1, digit):
print('c**k {}'.format(digit))
def __init__(self, Nway, Kshot, dataset, phase):
if dataset == 'cifar':
'''
use ciafr100 to train base predictors and meta-classifier
use cifar10 to test one-shot classification
'''
if phase == 'train':
self.train_data = Cifar('cifar100', 'train')
self.class_list = [i for i in range(80)]
elif phase == 'val':
self.train_data = Cifar('cifar100', 'train')
self.class_list = [i for i in range(80)]
elif phase == 'test_train':
self.train_data = Cifar('cifar10', 'train')
self.class_list = [i for i in range(10)]
elif phase == 'test_test':
self.train_data = Cifar('cifar10', 'test')
self.class_list = [i for i in range(10)]
else:
print(name_error)
elif dataset == 'imgnet':
'''
use miniImagenet 'train' and 'train_val' to train base predictors
'train' and 'train_val' contains 64 classes of mini imagenet
use 'val' to train meta classifier. val has 16 classes
use 'test' to test one-shot classification
'''
if phase == 'train':
self.train_data = MiniImageNet('train')
self.class_list = [i for i in range(64)]
elif phase == 'val':
self.train_data = MiniImageNet('val')
self.class_list = [i for i in range(16)]
elif phase == 'test_train':
self.train_data = MiniImageNet('test')
self.class_list = [i for i in range(20)]
elif phase == 'test_test':
self.train_data = MiniImageNet('test_val')
self.class_list = [i for i in range(20)]
else:
print(name_error)
elif dataset == 'mnist':
'''
use notMNIST 'train' to train base predictors
use notMNIST 'test' to train meta classifier
use MNIST 'train' and 'test' to test one-shot classification
'''
if phase == 'train':
self.train_data = MNIST('notMNIST', 'train')
elif phase == 'val':
self.train_data = MNIST('notMNIST', 'test')
elif phase == 'test_train':
self.train_data = MNIST('MNIST', 'train')
elif phase == 'test_test':
self.train_data = MNIST('MNIST', 'test')
else:
print(name_error)
self.class_list = [i for i in range(10)]
else:
print(dataset_name_error)
self.dataset = dataset
self.phase = phase
self.N = Nway
self.K = Kshot
self.train_subset = []
for i in self.class_list:
self.train_subset.append(
SubDataSet([i], (self.train_data.x, self.train_data.y)))
self.episode_set = []
train_step2 = tf.group(train_op, update_op)
acc2 = tf_acc(y_ph, y2)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
avg = Avg(desc=['train acc', 'train loss', 'lr', 'test acc'])
if args.dataset=='imgnet':
train_data = MiniImageNet('train')
val_data = MiniImageNet('train_val')
if args.dataset=='cifar':
train_data = Cifar('cifar100', 'train')
val_data = Cifar('cifar100', 'test')
if args.dataset=='mnist':
train_data = MNIST('notMNIST', 'train')
val_data = MNIST('notMNIST', 'test')
model.saver_init()
if args.from_ckpt:
model.saver_load(sess)
for i in range(1+args.initial_step, 1+args.max_iter):
batch_x, batch_y = train_data.next_batch(args.batch_size)
lr = (1e-0 if i < args.max_iter * 2.0 / 3.0 else 1e-1) * args.initial_lr
fd = {x_ph: batch_x, y_ph: batch_y, lr_ph: lr}
if sess.run(isTr):
p1, p2, _ = sess.run([acc, loss, train_step], fd)
else:
help='training epoch (default: 1)')
args = parser.parse_args()
#set epoch
epoch = args.epoch
#whether gpu is available
cuda = torch.cuda.is_available()
device = torch.device("cuda" if cuda else "cpu")
#create model
model = model_cnn().to(device)
model.train()
#create training dataset
training_dataset = MNIST(train_data, train_target)
#create validation dataset
validation_dataset = MNIST(validation_data, validation_target)
#create train loader
kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {}
train_loader = torch.utils.data.DataLoader(training_dataset,
batch_size=args.batch_size,
shuffle=True,
**kwargs)
#create validation loader
valid_loader = torch.utils.data.DataLoader(validation_dataset,
batch_size=args.batch_size,
shuffle=True,
import tensorflow_addons as tfa
from cnn import CnnSearchSpace
from config import AgingEvoConfig, TrainingConfig, BoundConfig
from dataset import MNIST
from search_algorithms import AgingEvoSearch
search_algorithm = AgingEvoSearch
training_config = TrainingConfig(
dataset=MNIST(),
epochs=30,
batch_size=128,
optimizer=lambda: tfa.optimizers.SGDW(learning_rate=0.005, momentum=0.9, weight_decay=4e-5),
callbacks=lambda: [],
)
search_config = AgingEvoConfig(
search_space=CnnSearchSpace(),
checkpoint_dir="artifacts/cnn_mnist"
)
bound_config = BoundConfig(
error_bound=0.035,
peak_mem_bound=2500,
model_size_bound=4500,
mac_bound=30000000
)
import nn
import executor.psgd as parallel
from codec.plain import Plain
from dataset import MNIST
from dataset.transforms import ImageCls, Shuffle
model = nn.model.SequentialModel(input_shape=[-1, 784])
model.add(nn.layer.Dense(128, activation=nn.activation.Tanh()))
model.add(nn.layer.Dense(128, activation=nn.activation.Tanh()))
model.add(nn.layer.Dense(10, activation=nn.activation.Softmax()))
model.setup(nn.loss.Cross_Entropy_With_Softmax(),
nn.metric.CategoricalAccuracy())
data = MNIST() # 使用MNIST数据集
trans = Shuffle().add(ImageCls()) # 先对数据集做Shuffle操作，再对数据集进行像素分类处理
job = parallel.ParallelSGD(model, data, trans)
nodes = parallel.parse_worker(worker_cnt=1)
# 两轮训练后，大约可以给出 'Loss': 0.262, 'accuracy': 0.954 的结果
print(job.parallel(nodes, codec=Plain, epoch=2))
running_loss = 0
with torch.no_grad():
for idx, x in enumerate(loader):
x = x.to(device)
recon_x, mu, log_sigma, sigma = model(x)
loss = ELBO(x, recon_x, mu, sigma)
running_loss += loss.item()
return running_loss / len(loader.dataset)
if __name__ == "__main__":
# Load dataset
print("Loading datasets.....")
start_time = time.time()
train_set = MNIST("data", split="train")
valid_set = MNIST("data", split="valid")
test_set = MNIST("data", split="test")
train_loader = DataLoader(train_set, batch_size=32, shuffle=True)
valid_loader = DataLoader(valid_set, batch_size=32, shuffle=False)
test_loader = DataLoader(test_set, batch_size=32, shuffle=False)
print("DONE in {:.2f} sec".format(time.time() - start_time))
do_training = False
do_testing = True
if do_training:
# Set hyperparameters
model = VAE(L=100)
t_loss, v_loss = train(model, valid_loader, test_loader)
torch.save(model.state_dict(), "vae.pth")