def main() -> None:
"""Start server and train five rounds."""
print(args)
assert (args.min_sample_size <= args.min_num_clients
), f"Num_clients shouldn't be lower than min_sample_size"
# Configure logger
fl.common.logger.configure("server", host=args.log_host)
# Load evaluation data
_, testset = utils.load_cifar(download=True)
# Create client_manager, strategy, and server
client_manager = fl.server.SimpleClientManager()
strategy = fl.server.strategy.FedAvg(
fraction_fit=args.sample_fraction,
min_fit_clients=args.min_sample_size,
min_available_clients=args.min_num_clients,
eval_fn=get_eval_fn(testset),
on_fit_config_fn=fit_config,
)
server = fl.server.Server(client_manager=client_manager, strategy=strategy)
# Run server
fl.server.start_server(
args.server_address,
server,
config={"num_rounds": args.rounds},
)
def main():
# Parse cmd arguments
hp = vars(utils.hyperparameters_from_args(hyperparameters_config, description='Fully connected neural network training on CIFAR-100'))
print(hp)
dataset = utils.load_cifar(USE_CIFAR100)
ops = build_graph(hp)
train(hp, dataset, ops)
def main(args):
train, valid, _ = load_cifar()
whiten, color = pca(train)
feat = args.features or int(np.sqrt(4 * K))
e = theanets.Experiment(
theanets.Autoencoder,
layers=(K, feat ** 2, K),
)
e.train(whiten(train), whiten(valid), input_noise=1)
plot_layers([
color(e.network.find(1, 0).get_value().T).T,
color(e.network.find('out', 0).get_value())], channels=3)
plt.tight_layout()
plt.show()
valid = whiten(valid[:100])
plot_images(color(valid), 121, 'Sample data', channels=3)
plot_images(color(e.network.predict(valid)), 122,
'Reconstructed data', channels=3)
plt.tight_layout()
plt.show()
def hyperparameter_search(hp_domain, n_iter, n_replicas, save_dir, model):
""" Random hyperparameter search on DNN model """
dataset = utils.load_cifar(USE_CIFAR100)
start_time = time.time()
gpu_devices = utils.get_available_gpus()
threads = []
for rep_id in range(n_replicas):
threads.append(threading.Thread(target=_hp_search_replica, kwargs={'start_time': start_time, 'gpu_devices': gpu_devices, 'hp_domain': hp_domain,
'save_dir': save_dir, 'dataset': dataset, 'rep_id': rep_id, 'n_iter': n_iter, 'model': model}))
threads[-1].start()
for thread in threads:
thread.join()
# Look for best hyperparameter set we found
best_results = {'acc': float('-inf'), 'model_name': None, 'hp': None}
for root, dirs, files in os.walk(save_dir):
for d in dirs:
if re.match('model_([0-9]+)_([0-9]+)', d) is not None:
results = np.load(os.path.join(root, d, 'results.npy')).tolist()
if best_results['acc'] < results['acc']:
best_results = results
best_results['model_name'] = d
print('\n\nHyperparameter search done!\n\tbest_acc=%4f\n\tbest_model_name=%s' % (best_results['acc'], best_results['model_name']) + '\n' + '#' * 100)
return best_results
def main(args):
train, valid, _ = load_cifar()
whiten, color = pca(train)
feat = args.features or int(np.sqrt(4 * K))
e = theanets.Experiment(
theanets.Autoencoder,
layers=(K, feat**2, K),
)
e.train(whiten(train), whiten(valid), input_noise=1)
plot_layers([
color(e.network.find(1, 0).get_value().T).T,
color(e.network.find('out', 0).get_value())
],
channels=3)
plt.tight_layout()
plt.show()
valid = whiten(valid[:100])
plot_images(color(valid), 121, 'Sample data', channels=3)
plot_images(color(e.network.predict(valid)),
122,
'Reconstructed data',
channels=3)
plt.tight_layout()
plt.show()
def main(args):
# 数据加载
(x_train, y_train), (x_test, y_test) = load_cifar(args.cifar_root)
# 随机选择训练样本
train_num = x_train.shape[0]
def next_batch(batch_size):
idx = np.random.choice(train_num, batch_size)
return x_train[idx], y_train[idx]
# 网络
vgg = VGG(image_size=32, name='vgg11')
opt = RmsProp(vgg.weights, lr=args.lr, decay=1e-3)
# 加载权重
if args.checkpoint:
weights = load_weights(args.checkpoint)
vgg.load_weights(weights)
print("load weights done")
# 评估
if args.eval_only:
indices = np.random.choice(len(x_test), args.eval_num, replace=False)
print('{} start evaluate'.format(
time.asctime(time.localtime(time.time()))))
acc = get_accuracy(vgg, x_test[indices], ys=y_test[indices])
print('{} acc on test dataset is :{:.3f}'.format(
time.asctime(time.localtime(time.time())), acc))
return
# 训练
num_steps = args.steps
for step in range(num_steps):
x, y_true = next_batch(args.batch_size)
# 前向传播
y_predict = vgg.forward(x.astype(np.float))
# print('y_pred: min{},max{},mean:{}'.format(np.min(y_predict, axis=-1),
# np.max(y_predict, axis=-1),
# np.mean(y_predict, axis=-1)))
# print('y_pred: {}'.format(y_predict))
acc = np.mean(
np.argmax(y_predict, axis=1) == np.argmax(y_true, axis=1))
# 计算loss
loss, gradient = cross_entropy_loss(y_predict, y_true)
# 反向传播
vgg.backward(gradient)
# 更新梯度
opt.iterate(vgg)
# 打印信息
print('{} step:{},loss:{:.4f},acc:{:.4f}'.format(
time.asctime(time.localtime(time.time())), step, loss, acc))
# 保存权重
if step % 100 == 0:
save_weights(
os.path.join(args.save_dir, 'weights-{:03d}.pkl'.format(step)),
vgg.weights)
def test(path):
(x_train, y_train), (x_test, y_test) = load_cifar(path)
print(x_train[0][0])
print(y_train[0])
vgg = VGG(name='vgg11')
import utils
utils.save_weights('./w.pkl', vgg.weights)
w = utils.load_weights('./w.pkl')
print(type(w))
print(w.keys())
def run_cifar(n_train=None, n_test=None, model_type="normal"):
datasize = {"n_train": n_train, "n_test": n_test}
transformer_params = {
"image_shape": 32,
"filter_shape_l1": 5, "step_shape_l1": 1, "n_l1_output": 16,
"filter_shape_l2": 5, "step_shape_l2": 1, "n_l2_output": 8,
"filter_shape_pooling": 8, "step_shape_pooling": 4
}
ensemble_params = {
"n_estimators" : 10,
"sampling_ratio" : 0.1,
"n_jobs" : -1
}
dataset = utils.load_cifar()
run(dataset, datasize, transformer_params, ensemble_params, model_type)
def main(args):
train, valid, _ = load_cifar()
e = theanets.Experiment(
theanets.Autoencoder,
layers=(3072, args.features ** 2, 3072))
e.train(train, valid)
plot_layers(e.network.weights, channels=3)
plt.tight_layout()
plt.show()
valid = valid[:100]
plot_images(valid, 121, 'Sample data', channels=3)
plot_images(e.network.predict(valid), 122, 'Reconstructed data', channels=3)
plt.tight_layout()
plt.show()
def main() -> None:
"""Load data, create and start CifarClient."""
parser = argparse.ArgumentParser(description="Flower")
parser.add_argument(
"--server_address",
type=str,
required=True,
help=f"gRPC server address",
)
parser.add_argument("--cid",
type=str,
required=True,
help="Client CID (no default)")
parser.add_argument(
"--log_host",
type=str,
help="Logserver address (no default)",
)
parser.add_argument(
"--data_dir",
type=str,
help="Directory where the dataset lives",
)
parser.add_argument(
"--model",
type=str,
default="ResNet18",
choices=["Net", "ResNet18"],
help="model to train",
)
args = parser.parse_args()
# Configure logger
fl.common.logger.configure(f"client_{args.cid}", host=args.log_host)
# model
model = utils.load_model(args.model)
model.to(DEVICE)
# load (local, on-device) dataset
trainset, testset = utils.load_cifar()
# Start client
client = CifarClient(args.cid, model, trainset, testset)
fl.client.start_client(args.server_address, client)
def main(args):
train, valid, _ = load_cifar()
e = theanets.Experiment(theanets.Autoencoder,
layers=(3072, args.features**2, 3072))
e.train(train, valid)
plot_layers(e.network.weights, channels=3)
plt.tight_layout()
plt.show()
valid = valid[:100]
plot_images(valid, 121, 'Sample data', channels=3)
plot_images(e.network.predict(valid),
122,
'Reconstructed data',
channels=3)
plt.tight_layout()
plt.show()
def main(features):
train, valid, _ = load_cifar()
whiten, color = pca(train[0])
feat = features or int(np.sqrt(2 * K))
n = theanets.Autoencoder([K, feat ** 2, K])
n.train(whiten(train), whiten(valid), input_noise=1, train_batches=313)
plot_layers([
color(n.find('hid1', 'w').get_value().T).T,
color(n.find('out', 'w').get_value())], channels=3)
plt.tight_layout()
plt.show()
valid = whiten(valid[:100])
plot_images(color(valid), 121, 'Sample data', channels=3)
plot_images(color(n.predict(valid)), 122,
'Reconstructed data', channels=3)
plt.tight_layout()
plt.show()
def main(features):
train, valid, _ = load_cifar()
whiten, color = pca(train[0])
feat = features or int(np.sqrt(2 * K))
n = theanets.Autoencoder([K, feat**2, K])
n.train(whiten(train), whiten(valid), input_noise=1, train_batches=313)
plot_layers([
color(n.find('hid1', 'w').get_value().T).T,
color(n.find('out', 'w').get_value())
],
channels=3)
plt.tight_layout()
plt.show()
valid = whiten(valid[:100])
plot_images(color(valid), 121, 'Sample data', channels=3)
plot_images(color(n.predict(valid)), 122, 'Reconstructed data', channels=3)
plt.tight_layout()
plt.show()
continue
X_ = X_train[(y_train == i) + (y_train == j)]
y_ = y_train[(y_train == i) + (y_train == j)]
X_transformed = train_model(X_)
print "pca " , i, " and " , j
plots[i, j].scatter(X_transformed[:, 0], X_transformed[:, 1], c=y_)
plots[i, j].set_xticks(())
plots[i, j].set_yticks(())
plots[j, i].scatter(X_transformed[:, 0], X_transformed[:, 1], c=y_)
plots[j, i].set_xticks(())
plots[j, i].set_yticks(())
if i == 0:
plots[i, j].set_title(j)
plots[j, i].set_ylabel(j)
plt.tight_layout()
plt.savefig(name)
if __name__ == '__main__':
train_set, valid_set, test_set = utils.load_MNIST(current_dir + "../mnist.pkl.gz")
data = np.asarray(np.vstack((train_set[0], valid_set[0], test_set[0])), dtype=np.float64)
y = np.hstack((train_set[1], valid_set[1], test_set[1]))
print 'loaded!'
train(data, y, "scatterplotMNIST.png")
trainx, trainy, testx, testy = utils.load_cifar(current_dir + "../cifar-10-batches-py/")
trainx = np.asarray(np.vstack((trainx, testx)), dtype=np.float64)
trainy = np.hstack((trainy, testy))
print "loaded!"
train(trainx, trainy, "scatterplotCIFAR.png")
new_test_acc=new_test_acc)
print(
'\tgeneralization_loss=%2.5f\tsub_loss=%2.5f\ttest_acc=%3.4f\tnew_test_acc=%3.4f'
% (mean_loss, mean_sub_loss, test_acc, new_test_acc))
mean_test_acc += test_acc / hp['steps']
mean_new_test_acc += new_test_acc / hp['steps']
if step >= hp['steps']:
break
print(
'\nTraining of error vectors done:\tmean_test_acc=%3.4f\tmean_new_test_acc=%3.4f'
% (mean_test_acc, mean_new_test_acc))
return mean_new_test_acc - mean_test_acc
def evaluate(probs, new_probs, batch_y_test):
test_acc = np.mean(np.equal(np.argmax(probs, axis=1), batch_y_test))
new_test_acc = np.mean(np.equal(np.argmax(new_probs, axis=1),
batch_y_test))
return test_acc, new_test_acc
if __name__ == '__main__':
hp = vars(
utils.hyperparameters_from_args(
hyperparameters_config,
description=
'Explicit generalization training of fully connected neural network on CIFAR-100'
))
dataset = utils.load_cifar(USE_CIFAR100)
explicit_generalization_training(hp, dataset)
image = image.transpose((2, 0, 1))
return image, target
trafos = [
utils.to_channel_first, utils.normalize, random_flip, random_color_jitter,
utils.to_tensor
]
trafos = functools.partial(utils.compose, transforms=trafos)
model = nn.Sequential(resnet18(num_classes=10), nn.LogSoftmax(dim=1))
model = model.to(device)
# get training and validation data
images, labels = utils.load_cifar('./cifar10/train')
(train_images, train_labels, val_images,
val_labels) = utils.make_cifar_train_val_split(images, labels)
train_dataset = utils.DatasetWithTransform(train_images,
train_labels,
transform=trafos)
val_dataset = utils.DatasetWithTransform(
val_images, val_labels, transform=utils.get_default_cifar_transform())
train_loader = DataLoader(
train_dataset,
batch_size=4,
shuffle=True,
)
val_loader = DataLoader(val_dataset, batch_size=25)
def __init__(self,
seq_len,
batch_size,
dataset='mnist',
set='train',
rng=None,
infinite=True,
digits=None):
if dataset == 'fashion_mnist':
(x_train, y_train), (x_test, y_test) = utils.load_fashion_mnist()
if set == 'train':
self.x = x_train
self.y = y_train
else:
self.x = x_test
self.y = y_test
elif dataset == 'mnist':
(x_train, y_train), (x_test, y_test) = utils.load_mnist()
if set == 'train':
self.x = x_train
self.y = y_train
elif set == 'test':
self.x = x_test
self.y = y_test
elif dataset == 'cifar10':
self.x, self.y = utils.load_cifar('data/cifar', subset=set)
self.x = np.transpose(self.x,
(0, 2, 3, 1)) # (N,3,32,32) -> (N,32,32,3)
self.x = np.float32(self.x)
self.img_shape = self.x.shape[1:]
self.input_dim = np.prod(self.img_shape)
else:
raise ValueError('wrong dataset name')
if dataset == 'mnist' or dataset == 'fashion_mnist':
self.input_dim = self.x.shape[-1]
self.img_shape = (int(np.sqrt(self.input_dim)),
int(np.sqrt(self.input_dim)), 1)
self.x = np.reshape(self.x, (self.x.shape[0], ) + self.img_shape)
self.x = np.float32(self.x)
self.classes = np.unique(self.y)
self.n_classes = len(self.classes)
self.y2idxs = {}
self.nsamples = 0
for i in list(self.classes):
self.y2idxs[i] = np.where(self.y == i)[0]
self.nsamples += len(self.y2idxs[i])
self.batch_size = batch_size
self.seq_len = seq_len
self.rng = np.random.RandomState(42) if not rng else rng
self.infinite = infinite
self.digits = digits if digits is not None else np.arange(
self.n_classes)
print(set, 'dataset size:', self.x.shape)
print(set, 'N classes', self.n_classes)
print(set, 'min, max', np.min(self.x), np.max(self.x))
print(set, 'nsamples', self.nsamples)
print(set, 'digits', self.digits)
print('--------------')
def main():
# Parse the arguments
args, device, model_name, paths, kwargs = parser.parserCIFAR()
print(args)
print('Name of the model:', model_name)
# Find the dataset
dataset = args.dataset
if Path('/dataset/' + dataset).is_dir():
print('Using folder /dataset')
path_to_dataset = './'
elif Path('/datasets2/' + dataset).is_dir():
path_to_dataset = '/datasets2/' + dataset
elif Path('../data').is_dir():
print('Looking for the dataset locally')
path_to_dataset = '../data'
else:
raise ValueError("No path to the dataset found")
save_path = './saved_models/{}/{}.pt'.format(dataset, model_name)
load_path = './saved_models/{}/{}.pt'.format(dataset, model_name)
results_path = './results/{}/{}.txt'.format(dataset, model_name)
if dataset == "cifar10":
train_loader, test_loader = utils.load_cifar(path_to_dataset, args, kwargs)
elif dataset == "mnist":
train_loader, test_loader = utils.load_mnist(path_to_dataset, args, kwargs)
else:
raise ValueError("Dataset {} not implemented".format(dataset))
# Case where there is no training
if args.visualize:
model = StandardNet(args) if args.standard else ProductNet(args)
model.load_state_dict(torch.load(load_path))
print('Model loaded')
model = model.to('cpu')
(visualize_standard_conv if args.standard else visualize)(model, model_name, args)
return
n_expe = args.experiments
results = np.zeros((n_expe, args.epochs))
for expe in range(n_expe):
model = StandardNet(args) if args.standard else ProductNet(args)
model = model.to(device)
optimizer = optim.Adam(model.parameters(), args.lr, weight_decay=1e-5)
if args.early_stopping:
earlyS = utils.EarlyStopping(mode='max', patience=args.early_stopping)
for epoch in range(1, args.epochs + 1):
train(args, model, device, train_loader, optimizer, epoch)
acc = test(model, device, test_loader)
results[expe, epoch - 1] = acc
stop = False if not args.early_stopping else earlyS.step(acc)
if stop:
print("Early stopping triggered at iteration", epoch)
print("Stopping the training.")
for i in range(epoch + 1, args.epochs + 1):
results[expe, i - 1] = acc
break
print('Experiment', expe, 'finished.')
if args.save_results or args.save_model:
utils.save_arguments(args.dataset, model_name)
if args.save_model:
torch.save(model.state_dict(), save_path)
if args.save_results:
np.savetxt(results_path, results)
average_acc = np.mean(results[:, -1])
print('All experiments done. Average accuracy:', average_acc)
return {'loss': -average_acc}
DATA_ROOT = './data/'
BATCH_LIST = [
'data_batch_1', 'data_batch_2', 'data_batch_3', 'data_batch_4',
'data_batch_5'
]
STORE_IMAGES = []
# Load the data
train_data = np.array([])
train_labels = np.array([])
# Create the Nearest Neighbour model
model = NearestNeighbour()
for index, batch in enumerate(BATCH_LIST):
data, labels = load_cifar(DATA_ROOT + batch)
X_train, X_test, y_train, y_test = train_test_split(data,
labels,
test_size=0.2)
model.train(X_train, y_train)
print("Batch %d: " % (index))
print(model.predict(X_test))
# Load the first batch only
# train_data, labels = load_cifar(DATA_ROOT + BATCH_LIST[0])
# image = train_data[2]
# red_channel = image[0:1024]
# red_channel = red_channel.reshape(32, 32)
# green_channel = image[1024: 2048]
# green_channel = green_channel.reshape(32, 32)
from model.Client import Client
from utils import load_mnist, sample_iid, load_cifar
from conf import Args
args = Args()
if not os.path.exists('data/results'):
os.makedirs('data/results')
if args.datatype == 'mnist':
path = './data/mnist'
train_data, test_data = load_mnist(path)
elif args.datatype == 'cifar':
path = './data/cifar'
train_data, test_data = load_cifar(path)
data_split = sample_iid(train_data, args.number_client)
print("model type: ", args.model_type)
print("dataset: ", args.datatype)
print("target test accuracy: ", args.target)
clients = []
for i in range(args.number_client):
client = Client(train_data, data_split[i], args)
clients.append(client)
server = Server(clients, test_data, args)
server.init_paras()
for i in range(1, d - 1):
trans(trans_blocks, trans_threads,
(S, T, Lx[i], Lz[i], iLx[i], iLz[i]))
conv3(conv_blocks, conv_threads, (S, S))
conv3(conv_blocks, conv_threads, (T, T))
trans(trans_blocks, trans_threads,
(S, T, Lx[-1], Lz[-1], iLx[-1], iLz[-1]))
if fix:
T -= S
return cp.mean(T) if gap else cp.trace(T.reshape(1024, 1024))
#Load CIFAR-10.
(X_train_all,
y_train_all), (X_test,
y_test) = load_cifar('/fs/vulcan-datasets/cifar-10-python')
for trial_i in range(5):
begin_time = time()
X_train = X_train_all[trial_i * args.num_train:(trial_i + 1) *
args.num_train]
y_train = y_train_all[trial_i * args.num_train:(trial_i + 1) *
args.num_train]
# X_test = X_test[:100]
# y_test = y_test[:100]
X = np.concatenate((X_train, X_test), axis=0)
N = X.shape[0]
N_train = X_train.shape[0]
N_test = X_test.shape[0]
X = cp.asarray(X).reshape(-1, 3, 1024)
#Calculate diagonal entries.
def main():
# -Set config-
args = argpars()
print('NETWORK:', args.net)
print('BATCH SIZE:', args.batch)
print('EPOCH:', args.epoch)
print('SIZE:', args.size)
print('GPU:', args.gpu)
print('TIMESTAMP:', timestamp)
# -End of Set config-
# =========================================================================
# -Load data
# Apply function that loads objective data
train_x, train_y, val_x, val_y = load_cifar()
# =========================================================================
# -Set num of class-
n_class = len(np.unique(train_y))
# -Model definition-
Net = archs[args.net]
model = L.Classifier(Net(n_class), lossfun=F.softmax_cross_entropy)
# -Select cupy or numpy-
if args.gpu > -1:
chainer.cuda.get_device(args.gpu).use()
model.to_gpu()
else:
xp = np
# -End of Select cupy or numpy-
# -Cast-
train_y = xp.array(train_y).astype(xp.int8)
val_y = xp.array(val_y).astype(xp.int8)
# -Set optimizer -
optimizer = chainer.optimizers.Adam()
optimizer.setup(model)
# -Preprocessing-
train = chainer.datasets.TupleDataset(PreprocessDataset(train_x, args.size), train_y)
val = chainer.datasets.TupleDataset(TestDataset(val_x, args.size), val_y)
# -Set iterators-
train_iter = chainer.iterators.SerialIterator(
train, args.batch)
val_iter = chainer.iterators.SerialIterator(
val, args.batch, repeat=False, shuffle=False)
# -Set Updater-
updater = chainer.training.updaters.StandardUpdater(
train_iter, optimizer, device=args.gpu)
# -Set trainer-
trainer = chainer.training.Trainer(updater, (args.epoch, 'epoch'))
# -Set trainer extensions-
# --Snapshot--
snapshot_name = 'snaptshot_{}'.format(timestamp)
snapshot_trigger = \
training.triggers.MinValueTrigger('main/loss', trigger=(1, 'epoch'))
trainer.extend(extensions.snapshot_object(
model.predictor, snapshot_name), trigger=snapshot_trigger)
# --End of Snapshot--
# --Evaluator--
trainer.extend(extensions.Evaluator(val_iter, model), trigger=(1, 'epoch'))
# --Report--
log_dir = 'logs'
log_name = 'log_{}'.format(timestamp)
log_name = os.path.join(log_dir, log_name)
trainer.extend(extensions.LogReport(
['epoch',
'main/accuracy',
'validation/main/accuracy',
'elapsed_time'],
log_name=log_name))
trainer.extend(extensions.PrintReport(
['epoch',
'main/accuracy',
'validation/main/accuracy',
'elapsed_time']))
trainer.extend(extensions.ProgressBar(update_interval=10))
# --End of Report--
# -End of Set trainer extensions-
# -Start learning-
trainer.run()
# -Serialize model-
model.to_cpu()
model_name = '{}_{}.npz'.format(timestamp, args.net)
model_name = os.path.join('result', 'model_npzs', model_name)
chainer.serializers.save_npz(model_name, model.predictor)
# -End of Serialize model-
# -Rename snapshot-
os.rename('result/' + snapshot_name, 'result/snapshots/' + snapshot_name)
return 0
device = torch.device('cuda')
else:
print("GPU is not available, training will run on the CPU")
device = torch.device('cpu')
# run this in google colab to get the utils.py file
# we will reuse the training function, validation function and
# data preparation from the previous notebook
cifar_dir = './cifar10'
categories = os.listdir('./cifar10/train')
categories.sort()
images, labels = utils.load_cifar(os.path.join(cifar_dir, 'train'))
(train_images, train_labels, val_images,
val_labels) = utils.make_cifar_train_val_split(images, labels)
"""## Data Augmentation
The goal of data augmentation is to increase the amount of training data by transforming the input images in a way that they still resemble realistic images, but differ from the input to the transformation.
Here, we will start with two transformations:
- random flips along the vertical centerline
- random color jitters
"""
# define random augmentations
def random_flip(image, target, probability=.5):
""" Randomly mirror the image across the vertical axis.
import os
import numpy as np
import sys
sys.path.append(os.path.abspath('utils'))
import matplotlib.pyplot as plt
import torch
import utils
from functools import partial
from logistic_regressor_conv_filters import gaussian_kernel
laplacian_filter = torch.Tensor([[0,1,0], [1,-4,1], [0,1,0]])
gaussian_filter = gaussian_kernel(dim=3)
imgs, labels = utils.load_cifar(os.path.join('./cifar10', 'test'))
trafos = [utils.to_channel_first, utils.normalize, utils.to_tensor]
trafos = partial(utils.compose, transforms=trafos)
data = utils.DatasetWithTransform(imgs, labels, transform=trafos)
loader = torch.utils.data.DataLoader(data, batch_size=4, shuffle=False)
conv_gaussian = torch.nn.Conv2d(in_channels=3, out_channels=6, kernel_size=3, bias=False)
conv_gaussian.weight.data[:,:] = gaussian_filter
conv_laplacian = torch.nn.Conv2d(in_channels=6, out_channels=9, kernel_size=3, bias=False)
conv_laplacian.weight.data[:,:] = laplacian_filter
conv_laplacian_alone = torch.nn.Conv2d(in_channels=3, out_channels=6, kernel_size=3, bias=False)
conv_laplacian_alone.weight.data[:,:] = laplacian_filter
from sklearn.model_selection import GridSearchCV
from keras.wrappers.scikit_learn import KerasClassifier
from preact_resnet_model import ResNet_model
import utils
def create_model(res_layer_params=(3, 32, 25), reg=0.0001, optimizer='adam'):
model = ResNet_model(res_layer_params=res_layer_params, reg=reg)
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
return model
(X_train, y_train), (X_test, y_test) = utils.load_cifar(normalize=True)
model = KerasClassifier(build_fn=create_model, verbose=2)
batch_size = [32, 64, 128]
epochs = [50, 100, 200]
res_layer_params = [(3, 32, 25), (3, 32, 10)]
reg = [0.0001, 0.0005, 0.001, 0.005]
optimizer = ['SGD', 'RMSprop', 'Adam']
params_grid = dict(batch_size=batch_size,
epochs=epochs,
optimizer=optimizer,
res_layer_params=res_layer_params,
reg=reg)
grid = GridSearchCV(estimator=model, param_grid=params_grid, cv=5, n_jobs=8)
grid_result = grid.fit(X_train, y_train)
is_gpu = torch.cuda.is_available()
if is_gpu:
print("GPU Available")
device = torch.device('cuda')
else:
print("GPU not available")
device = torch.device('cpu')
categories = os.listdir('./cifar10/train')
categories.sort()
print(categories)
cifar_dir = './cifar10'
images, labels = utils.load_cifar(os.path.join(cifar_dir, 'train'))
n_imgs = len(images)
# train_test_split from sklearn shuffles and stratifies the data such that
# the same number of samples per classes is present in train and val splits
(train_imgs, val_imgs, train_labels,
val_labels) = sklearn.model_selection.train_test_split(images,
labels,
shuffle=True,
test_size=.15,
stratify=labels)
assert len(train_imgs) == len(train_labels)
assert len(val_imgs) == len(val_labels)
metavar='DIR',
default='./figures/',
help='Path to saved figure')
parser.add_argument('--num-data',
type=int,
default=5000,
help='number of training data')
parser.add_argument('--seed', type=int, default=1234, help='Random seed')
args = parser.parse_args()
cifar_path = args.cifar_root
fig_path = args.fig_path
num_data = args.num_data
SEED = args.seed
X, y = load_cifar(cifar_path) # load data
X_svm, y_svm = X[:num_data], y[:num_data] # select only 5000 data
X_train, X_test, y_train, y_test = train_test_split(
X_svm, y_svm, test_size=0.2, random_state=SEED,
stratify=y_svm) # split train/test set
print('=' * 10, 'Implementing PCA on data', '=' * 10)
# PCA using sklearn's Pipeline
# by applying standard scaler (shift data to zero mean, unit variance) before PCA
pca_010 = Pipeline([('scaling', StandardScaler()),
('pca', PCA(n_components=int(0.1 * X_train.shape[1])))
]).fit(X_train) # 10% of components
pca_030 = Pipeline([('scaling', StandardScaler()),
('pca', PCA(n_components=int(0.3 * X_train.shape[1])))
]).fit(X_train) # 30% of components
def main(args):
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = args.gpu_fraction
set_session(tf.Session(config=config))
if (not os.path.exists(args.output_dir)):
os.makedirs(args.output_dir)
(X_train, y_train), (X_test, y_test) = utils.load_cifar(args.normalize)
train_mode, eval_mode = utils.parse_mode(args.mode)
if (args.mode == 'eval' and args.from_pretrain == None):
print("from_pretrain argument needs to be specified in eval mode")
return
model = ResNet_model(res_layer_params=(3, 32, args.resnet_depth),
reg=args.regularization)
model.summary()
#load model from pretrain
if (args.from_pretrain != None):
print("Loading weight from pretrain: {}".format(args.from_pretrain))
model.load_weights(args.from_pretrain)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
#train model
if (train_mode):
save_checkpoint = ModelCheckpoint(args.output_dir + "/model.h5",
monitor='val_acc',
save_best_only=True,
mode='max')
early_stop = EarlyStopping(monitor='val_loss', patience=50, verbose=1)
if (args.epochs == 100):
lr_scheduler = LearningRateScheduler(lr_schedule_100)
elif (args.epochs == 200):
lr_scheduler = LearningRateScheduler(lr_schedule_200)
start_time = time.time()
history = model.fit(
X_train,
y_train,
validation_data=(X_test, y_test),
epochs=100,
batch_size=args.batch_size,
callbacks=[early_stop, lr_scheduler, save_checkpoint])
end_time = time.time()
print("Model took {} seconds to train".format(end_time - start_time))
utils.plot_model_history(history, args.output_dir)
#evaluate model
if (eval_mode):
if (train_mode):
model.load_weights(args.output_dir + '/model.h5')
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
scores = model.evaluate(X_test, y_test)
print("{}: {}, {}: {}".format(model.metrics_names[0], scores[0],
model.metrics_names[1], scores[1]))
for i in range(1, d - 1):
trans(trans_blocks, trans_threads,
(S, T, Lx[i], Lz[i], iLx[i], iLz[i]))
conv3(conv_blocks, conv_threads, (S, S))
conv3(conv_blocks, conv_threads, (T, T))
trans(trans_blocks, trans_threads,
(S, T, Lx[-1], Lz[-1], iLx[-1], iLz[-1]))
if fix:
T -= S
return cp.mean(T) if gap else cp.trace(T.reshape(1024, 1024))
#Load CIFAR-10.
(X_train, y_train), (X_test, y_test) = load_cifar()
X = np.concatenate((X_train, X_test), axis=0)
N = X.shape[0]
N_train = X_train.shape[0]
N_test = X_test.shape[0]
X = cp.asarray(X).reshape(-1, 3, 1024)
#Calculate diagonal entries.
L = []
iL = []
for i in range(N):
Lx, iLx = xx(X[i])
L.append(Lx)
iL.append(iLx)
#####Calculate kernel values.
