def predict_with_pre_trained_model_example():
# Define a model.
net = build_network(nn.Sequential())
# Load the model parameters.
net.load_parameters('./net.params')
transformer = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(0.13, 0.31)])
# Predict the first six images in the validation dataset.
mnist_valid = datasets.FashionMNIST(train=False)
X, y = mnist_valid[:6]
preds = []
for x in X:
x = transformer(x).expand_dims(axis=0)
pred = net(x).argmax(axis=1)
preds.append(pred.astype('int32').asscalar())
# Visualize the images and compare the prediction with the ground truth.
_, figs = plt.subplots(1, 6, figsize=(15, 15))
text_labels = [
't-shirt', 'trouser', 'pullover', 'dress', 'coat', 'sandal', 'shirt',
'sneaker', 'bag', 'ankle boot'
]
for f, x, yi, pyi in zip(figs, X, y, preds):
f.imshow(x.reshape((28, 28)).asnumpy())
ax = f.axes
ax.set_title(text_labels[yi] + '\n' + text_labels[pyi])
ax.title.set_fontsize(20)
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.show()
def prepare_data(batch_size):
mnist_train = datasets.FashionMNIST(train=True)
X, y = mnist_train[0]
("X shape: ", X.shape, "X dtype", X.dtype, "y:", y)
text_labels = [
"t-shirt",
"trouser",
"pullover",
"dress",
"coat",
"sandal",
"shirt",
"sneaker",
"bag",
"ankle boot",
]
X, y = mnist_train[0:10]
transformer = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(0.13, 0.31)])
mnist_train = mnist_train.transform_first(transformer)
train_data = gluon.data.DataLoader(mnist_train,
batch_size=batch_size,
shuffle=True,
num_workers=4)
mnist_valid = gluon.data.vision.FashionMNIST(train=False)
valid_data = gluon.data.DataLoader(
mnist_valid.transform_first(transformer),
batch_size=batch_size,
num_workers=4)
return train_data, valid_data
def prepare_data(batch_size):
mnist_train = datasets.FashionMNIST(train=True)
transformer = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(0.286, 0.352)])
mnist_train = mnist_train.transform_first(transformer)
train_data = gluon.data.DataLoader(mnist_train,
batch_size=batch_size,
shuffle=True,
num_workers=4)
return train_data
def ready_data():
transformer = transforms.Compose([transforms.ToTensor(),transforms.Normalize(0.13, 0.31)])
mnist_train = datasets.FashionMNIST(train=True)
mnist_train = mnist_train.transform_first(transformer)
batch_size = 256
train_data = gluon.data.DataLoader(mnist_train, batch_size=batch_size, shuffle=True)
for data, label in train_data:
print(data.shape, label.shape)
break
mnist_valid = gluon.data.vision.FashionMNIST(train=False)
valid_data = gluon.data.DataLoader(mnist_valid.transform_first(transformer),batch_size=batch_size)
return train_data, valid_data, batch_size
def get_train_data():
mnist_train = datasets.FashionMNIST(train=True)
# X, y = mnist_train[0]
# X, y = mnist_train[0:10]
# # plot images
# display.set_matplotlib_formats('svg')
# _, figs = plt.subplots(1, X.shape[0], figsize=(15, 15))
# for f,x,yi in zip(figs, X,y):
# # 3D->2D by removing the last channel dim
# f.imshow(x.reshape((28,28)).asnumpy())
# ax = f.axes
# ax.set_title(text_labels[int(yi)])
# ax.title.set_fontsize(14)
# ax.get_xaxis().set_visible(False)
# ax.get_yaxis().set_visible(False)
# plt.show()
return mnist_train
def get_data():
mnist_train = datasets.FashionMNIST(train=True)
X, y = mnist_train[0]
('X shape: ', X.shape, 'X dtype', X.dtype, 'y:', y)
text_labels = [
't-shirt', 'trouser', 'pullover', 'dress', 'coat',
'sandal', 'shirt', 'sneaker', 'bag', 'ankle boot']
X, y = mnist_train[0:6]
# plot images
_, figs = plt.subplots(1, X.shape[0], figsize=(15, 15))
for f,x,yi in zip(figs, X,y):
# 3D->2D by removing the last channel dim
f.imshow(x.reshape((28,28)).asnumpy())
ax = f.axes
ax.set_title(text_labels[int(yi)])
ax.title.set_fontsize(20)
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.show()
return True
def main():
net = nn.Sqeuential()
net.add(nn.Conv2D(channels=6, kernel_size=5, activation='relu'),
nn.MaxPool2D(pool_size=2, strides=2),
nn.Conv2D(channels=16, kernel_size=3, activation='relu'),
nn.MaxPool2D(pool_size=2, strides=2), nn.Flatten(),
nn.Dense(120, activation='relu'), nn.Dense(84, activation='relu'),
nn.Dense(10))
net.load_parameters('net.params')
transformer = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(0.13, 0.31)])
mnist_valid = datasets.FashionMNIST(train=False)
X, y = mnist_valid[:10]
preds = []
for x in X:
x = transformer(x).expand_dims(axis=0)
pred = net(x).argmax(axis=1)
preds.append(pred.astype('int32').asscalar())
import mxnet as mx
import matplotlib.pyplot as plt
from mxnet import autograd, init, gluon, gpu, cpu, context
from mxnet.gluon.data.vision import datasets, transforms
from simplenet import SimpleNet
text_labels = [
't-shirt', 'trouser', 'pullover', 'dress', 'coat', 'sandal', 'shirt',
'sneaker', 'bag', 'ankle boot'
]
mnist_train = datasets.FashionMNIST(train=True)
# Check the image and the label
X, y = mnist_train[0]
class_name = text_labels[y]
print("X shape: {}, X dtype:{}".format(X.shape, X.dtype))
print("y: {}({}), y dtype:{}".format(y, class_name, y.dtype))
plt.figure(class_name)
plt.imshow(X.squeeze(axis=2).asnumpy())
plt.show()
# Define the transformation and load the data into batches
transformer = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(0.13, 0.31),
])
batch_size = 256
train_data = gluon.data.DataLoader(mnist_train.transform_first(transformer),
net.add(nn.Conv2D(channels=6, kernel_size=5, activation='relu'),
nn.MaxPool2D(pool_size=2, strides=2),
nn.Conv2D(channels=16, kernel_size=3, activation='relu'),
nn.MaxPool2D(pool_size=2, strides=2), nn.Flatten(),
nn.Dense(120, activation='relu'), nn.Dense(84, activation='relu'),
nn.Dense(10))
net.load_params('net.params')
# predict
transformer = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(0.13, 0.31)])
mnist_valid = datasets.FashionMNIST(train=False)
X, y = mnist_valid[:6]
preds = []
for x in X:
x = transformer(x).expand_dims(axis=0)
pred = net(x).argmax(axis=1)
preds.append(pred.astype('int32').asscalar())
_, figs = plt.subplots(1, 6, figsize=(15, 15))
text_labels = [
't-shirt', 'trouser', 'pullover', 'dress', 'coat', 'sanda', 'shirt',
'sneaker', 'bag', 'ankle boot'
]
for f, x, yi, pyi in zip(figs, X, y, preds):
f.imshow(x.reshape((28, 28)).asnumpy())
def train():
mnist_train = datasets.FashionMNIST(train=True)
X, y = mnist_train[0]
print('X shape: ', X.shape, 'X dtype', X.dtype, 'y:', y)
transformer = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(0.13, 0.31)])
mnist_train = mnist_train.transform_first(transformer)
batch_size = 256
train_data = gluon.data.DataLoader(
mnist_train, batch_size=batch_size, shuffle=True, num_workers=4)
for data, label in train_data:
print(data.shape, label.shape)
break
mnist_valid = gluon.data.vision.FashionMNIST(train=False)
valid_data = gluon.data.DataLoader(
mnist_valid.transform_first(transformer),
batch_size=batch_size, num_workers=4)
net = nn.HybridSequential()
net.add(nn.Conv2D(channels=6, kernel_size=5, activation='relu'),
nn.MaxPool2D(pool_size=2, strides=2),
nn.Conv2D(channels=16, kernel_size=3, activation='relu'),
nn.MaxPool2D(pool_size=2, strides=2),
nn.Flatten(),
nn.Dense(120, activation="relu"),
nn.Dense(84, activation="relu"),
nn.Dense(10))
net.initialize(init=init.Xavier(), ctx=device)
net.hybridize()
softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()
trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1})
for epoch in range(10):
train_loss, train_acc, valid_acc = 0., 0., 0.
tic = time.time()
for data, label in train_data:
data = data.copyto(device)
label = label.copyto(device)
# forward + backward
with autograd.record():
output = net(data)
loss = softmax_cross_entropy(output, label)
loss.backward()
# update parameters
trainer.step(batch_size)
# calculate training metrics
train_loss += loss.mean().asscalar()
train_acc += acc(output, label)
# calculate validation accuracy
for data, label in valid_data:
data = data.copyto(device)
label = label.copyto(device)
valid_acc += acc(net(data), label)
print("Epoch %d: loss %.3f, train acc %.3f, test acc %.3f, in %.1f sec" % (
epoch, train_loss/len(train_data), train_acc/len(train_data),
valid_acc/len(valid_data), time.time()-tic))
net.export('net-trained', epoch=10)
def run_mnist(
hook=None,
set_modes=False,
num_steps_train=None,
num_steps_eval=None,
epochs=2,
save_interval=None,
save_path="./saveParams",
):
batch_size = 4
normalize_mean = 0.13
mnist_train = datasets.FashionMNIST(train=True)
X, y = mnist_train[0]
("X shape: ", X.shape, "X dtype", X.dtype, "y:", y)
text_labels = [
"t-shirt",
"trouser",
"pullover",
"dress",
"coat",
"sandal",
"shirt",
"sneaker",
"bag",
"ankle boot",
]
transformer = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(normalize_mean, 0.31)])
mnist_train = mnist_train.transform_first(transformer)
mnist_valid = gluon.data.vision.FashionMNIST(train=False)
train_data = gluon.data.DataLoader(mnist_train,
batch_size=batch_size,
shuffle=True,
num_workers=4)
valid_data = gluon.data.DataLoader(
mnist_valid.transform_first(transformer),
batch_size=batch_size,
num_workers=4)
# Create Model in Gluon
net = nn.HybridSequential()
net.add(
nn.Conv2D(channels=6, kernel_size=5, activation="relu"),
nn.MaxPool2D(pool_size=2, strides=2),
nn.Conv2D(channels=16, kernel_size=3, activation="relu"),
nn.MaxPool2D(pool_size=2, strides=2),
nn.Flatten(),
nn.Dense(120, activation="relu"),
nn.Dense(84, activation="relu"),
nn.Dense(10),
)
net.initialize(init=init.Xavier(), ctx=mx.cpu())
if hook is not None:
# Register the forward Hook
hook.register_hook(net)
softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()
trainer = gluon.Trainer(net.collect_params(), "sgd",
{"learning_rate": 0.1})
hook.register_hook(softmax_cross_entropy)
# Start the training.
for epoch in range(epochs):
train_loss, train_acc, valid_acc = 0.0, 0.0, 0.0
tic = time.time()
if set_modes:
hook.set_mode(modes.TRAIN)
i = 0
for data, label in train_data:
data = data.as_in_context(mx.cpu(0))
# forward + backward
with autograd.record():
output = net(data)
loss = softmax_cross_entropy(output, label)
loss.backward()
# update parameters
trainer.step(batch_size)
# calculate training metrics
train_loss += loss.mean().asscalar()
train_acc += acc(output, label)
i += 1
if num_steps_train is not None and i >= num_steps_train:
break
# calculate validation accuracy
if set_modes:
hook.set_mode(modes.EVAL)
i = 0
for data, label in valid_data:
data = data.as_in_context(mx.cpu(0))
val_output = net(data)
valid_acc += acc(val_output, label)
loss = softmax_cross_entropy(val_output, label)
i += 1
if num_steps_eval is not None and i >= num_steps_eval:
break
print(
"Epoch %d: loss %.3f, train acc %.3f, test acc %.3f, in %.1f sec" %
(
epoch,
train_loss / len(train_data),
train_acc / len(train_data),
valid_acc / len(valid_data),
time.time() - tic,
))
if save_interval is not None and (epoch % save_interval) == 0:
net.save_parameters("{0}/params_{1}.params".format(
save_path, epoch))
def main():
mnist_train = datasets.FashionMNIST(train=True)
X, y = mnist_train[0]
#print('X ',X)
#print('y ', y)
print('X shape: ', X.shape, 'X dtype ', X.dtype, 'y:', y)
text_labels = [
't-shirt', 'trouser', 'pullover', 'dress', 'coat', 'sandal', 'shirt',
'sneaker', 'bag', 'ankle boot'
]
X, y = mnist_train[0:10]
# plot images
# display.set_matplotlib_formats('svg')
# _, figs = plt.subplots(1, X.shape[0], figsize=(15, 15))
# for f,x,yi in zip(figs, X,y):
# # 3D->2D by removing the last channel dim
# f.imshow(x.reshape((28,28)).asnumpy())
# ax = f.axes
# ax.set_title(text_labels[int(yi)])
# ax.title.set_fontsize(14)
# ax.get_xaxis().set_visible(False)
# ax.get_yaxis().set_visible(False)
# plt.show()
#
transformer = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(0.13, 0.31)])
mnist_train = mnist_train.transform_first(transformer)
batch_size = 64
train_data = gluon.data.DataLoader(mnist_train,
batch_size=batch_size,
shuffle=True,
num_workers=4)
mnist_valid = gluon.data.vision.FashionMNIST(train=False)
valid_data = gluon.data.DataLoader(
mnist_valid.transform_first(transformer),
batch_size=batch_size,
num_workers=4)
net = nn.Sequential()
net.add(nn.Conv2D(channels=6, kernel_size=5, activation='relu'),
nn.MaxPool2D(pool_size=2, strides=2),
nn.Conv2D(channels=16, kernel_size=3, activation='relu'),
nn.MaxPool2D(pool_size=2, strides=2), nn.Flatten(),
nn.Dense(120, activation='relu'), nn.Dense(84, activation='relu'),
nn.Dense(10))
net.initialize(init=init.Xavier())
softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()
trainer_opt = gluon.Trainer(net.collect_params(), 'sgd',
{'learning_rate': 0.1})
for epoch in range(10):
train_loss, train_acc, valid_acc = 0., 0., 0.,
tic = time.time()
for data, label in train_data:
# forward and backward
with autograd.record():
output = net(data)
loss = softmax_cross_entropy(output, label)
loss.backward()
# update parameters
trainer_opt.step(batch_size)
# calculate training metrics
train_loss += loss.mean().asscalar()
train_acc += acc(output, label)
for data, label in valid_data:
valid_acc += acc(net(data), label)
print(
"epoch %d: loss %.3f, train acc %.3f, test acc %.3f, in %.1f sec" %
(epoch, train_loss / len(train_data), train_acc / len(train_data),
valid_acc / len(valid_data), time.time() - tic))
net.save_parameters('net.params')
def train_neural_network_example():
mnist_train = datasets.FashionMNIST(train=True)
X, y = mnist_train[0]
print('X shape:', X.shape, 'X dtype:', X.dtype, 'y:', y)
text_labels = [
't-shirt', 'trouser', 'pullover', 'dress', 'coat', 'sandal', 'shirt',
'sneaker', 'bag', 'ankle boot'
]
X, y = mnist_train[0:6]
# Plot images.
_, figs = plt.subplots(1, X.shape[0], figsize=(15, 15))
for f, x, yi in zip(figs, X, y):
# 3D -> 2D by removing the last channel dim.
f.imshow(x.reshape((28, 28)).asnumpy())
ax = f.axes
ax.set_title(text_labels[int(yi)])
ax.title.set_fontsize(20)
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.show()
transformer = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(0.13, 0.31)])
mnist_train = mnist_train.transform_first(transformer)
batch_size = 256
train_data = gluon.data.DataLoader(mnist_train,
batch_size=batch_size,
shuffle=True,
num_workers=4)
for data, label in train_data:
print(data.shape, label.shape)
break
mnist_valid = gluon.data.vision.FashionMNIST(train=False)
valid_data = gluon.data.DataLoader(
mnist_valid.transform_first(transformer),
batch_size=batch_size,
num_workers=4)
# Define a model.
net = build_network(nn.Sequential())
net.initialize(init=init.Xavier())
#net.collect_params().initialize(init.Xavier(), ctx=ctx)
# Define the loss function and optimization method for training.
softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()
trainer = gluon.Trainer(net.collect_params(), 'sgd',
{'learning_rate': 0.1})
#trainer = gluon.Trainer(net.collect_params(), 'adam', {'learning_rate': .001})
train_model(net,
trainer,
softmax_cross_entropy,
train_data,
valid_data,
num_epochs=10)
# Save the model parameters.
net.save_parameters('./net.params')
def run_mnist_gluon_model(
hook=None,
hybridize=False,
set_modes=False,
register_to_loss_block=False,
num_steps_train=None,
num_steps_eval=None,
make_input_zero=False,
normalize_mean=0.13,
normalize_std=0.31,
save_custom_tensor=False,
):
batch_size = 4
if make_input_zero:
mnist_train = datasets.FashionMNIST(
train=True,
transform=lambda data, label: (data.astype(np.float32) * 0, label))
normalize_mean = 0.0
else:
mnist_train = datasets.FashionMNIST(train=True)
X, y = mnist_train[0]
("X shape: ", X.shape, "X dtype", X.dtype, "y:", y)
text_labels = [
"t-shirt",
"trouser",
"pullover",
"dress",
"coat",
"sandal",
"shirt",
"sneaker",
"bag",
"ankle boot",
]
X, y = mnist_train[0:10]
transformer = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(normalize_mean, 0.31)])
mnist_train = mnist_train.transform_first(transformer)
mnist_valid = gluon.data.vision.FashionMNIST(train=False)
train_data = gluon.data.DataLoader(mnist_train,
batch_size=batch_size,
shuffle=True,
num_workers=4)
valid_data = gluon.data.DataLoader(
mnist_valid.transform_first(transformer),
batch_size=batch_size,
num_workers=4)
# Create Model in Gluon
net = nn.HybridSequential()
net.add(
nn.Conv2D(channels=6, kernel_size=5, activation="relu"),
nn.MaxPool2D(pool_size=2, strides=2),
nn.Conv2D(channels=16, kernel_size=3, activation="relu"),
nn.MaxPool2D(pool_size=2, strides=2),
nn.Flatten(),
nn.Dense(120, activation="relu"),
nn.Dense(84, activation="relu"),
nn.Dense(10),
)
net.initialize(init=init.Xavier(), ctx=mx.cpu())
if hybridize:
net.hybridize(())
if hook is not None:
# Register the forward Hook
hook.register_hook(net)
softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()
trainer = gluon.Trainer(net.collect_params(), "sgd",
{"learning_rate": 0.1})
if register_to_loss_block:
hook.register_hook(softmax_cross_entropy)
if set_modes:
train_loss_name = eval_loss_name = "loss_scalar"
train_acc_name = eval_acc_name = "acc"
else:
train_loss_name = "train_loss_scalar"
eval_loss_name = "eval_loss_scalar"
train_acc_name = "train_acc"
eval_acc_name = "loss_acc"
# Start the training.
if save_custom_tensor:
hook.save_tensor("custom_tensor_1", mx.nd.array([1, 2, 3]))
for epoch in range(1):
train_loss, train_acc, valid_acc = 0.0, 0.0, 0.0
tic = time.time()
if set_modes:
hook.set_mode(modes.TRAIN)
i = 0
for data, label in train_data:
if save_custom_tensor:
hook.save_tensor("custom_tensor_2", mx.nd.array([1, 2, 3]))
data = data.as_in_context(mx.cpu(0))
# forward + backward
with autograd.record():
output = net(data)
loss = softmax_cross_entropy(output, label)
loss.backward()
# update parameters
trainer.step(batch_size)
# calculate training metrics
train_loss += loss.mean().asscalar()
train_acc += acc(output, label)
# hook.save_scalar(train_loss_name, train_loss)
# hook.save_scalar(train_acc_name, train_acc)
if save_custom_tensor:
# This tensor will not be added to default collections since
# collections have already been exported
hook.save_tensor("custom_tensor_3", mx.nd.array([1, 2, 3]))
i += 1
if num_steps_train is not None and i >= num_steps_train:
break
# calculate validation accuracy
if set_modes:
hook.set_mode(modes.EVAL)
i = 0
for data, label in valid_data:
data = data.as_in_context(mx.cpu(0))
val_output = net(data)
valid_acc += acc(val_output, label)
loss = softmax_cross_entropy(val_output, label)
# hook.save_tensor('eval_labels', label)
# hook.save_scalar(eval_acc_name, valid_acc)
# hook.save_scalar(eval_loss_name, loss)
i += 1
if num_steps_eval is not None and i >= num_steps_eval:
break
print(
"Epoch %d: loss %.3f, train acc %.3f, test acc %.3f, in %.1f sec" %
(
epoch,
train_loss / len(train_data),
train_acc / len(train_data),
valid_acc / len(valid_data),
time.time() - tic,
))
def basic_example():
# Allocate data to a GPU.
# MXNet's NDArray is very similar to Numpy.
# One major difference is NDArray has a context attribute that specifies which device this array is on.
# By default, it is cpu().
x = nd.ones((3, 4), ctx=gpu())
# Copy x to the second GPU, gpu(1).
x.copyto(gpu(1))
# Run an operation on a GPU.
y = nd.random.uniform(shape=(3, 4), ctx=gpu())
print('x + y =', x + y)
# Run a neural network on a GPU.
net = nn.Sequential()
net.add(nn.Conv2D(channels=6, kernel_size=5, activation='relu'),
nn.MaxPool2D(pool_size=2, strides=2),
nn.Conv2D(channels=16, kernel_size=3, activation='relu'),
nn.MaxPool2D(pool_size=2, strides=2),
nn.Flatten(),
nn.Dense(120, activation='relu'),
nn.Dense(84, activation='relu'),
nn.Dense(10))
# Load the saved parameters into GPU 0 directly, or use net.collect_params().reset_ctx to change the device.
net.load_parameters('./net.params', ctx=gpu(0))
# Create input data on GPU 0.
x = nd.random.uniform(shape=(1,1,28,28), ctx=gpu(0))
# The forward function will then run on GPU 0.
print('net(x) =', net(x))
# Multi-GPU training.
batch_size = 256
transformer = transforms.Compose([transforms.ToTensor(), transforms.Normalize(0.13, 0.31)])
train_data = gluon.data.DataLoader(datasets.FashionMNIST(train=True).transform_first(transformer), batch_size, shuffle=True, num_workers=4)
valid_data = gluon.data.DataLoader(datasets.FashionMNIST(train=False).transform_first(transformer), batch_size, shuffle=False, num_workers=4)
# Diff 1: Use two GPUs for training.
devices = [gpu(0), gpu(1)]
# Diff 2: reinitialize the parameters and place them on multiple GPUs.
net.collect_params().initialize(force_reinit=True, ctx=devices)
# Loss and trainer are the same as before.
softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()
trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1})
for epoch in range(10):
train_loss = 0.
tic = time()
for data, label in train_data:
# Diff 3: split batch and load into corresponding devices.
data_list = gluon.utils.split_and_load(data, devices)
label_list = gluon.utils.split_and_load(label, devices)
# Diff 4: run forward and backward on each devices.
# MXNet will automatically run them in parallel.
with autograd.record():
losses = [softmax_cross_entropy(net(X), y) for X, y in zip(data_list, label_list)]
for l in losses:
l.backward()
trainer.step(batch_size)
# Diff 5: sum losses over all devices.
train_loss += sum([l.sum().asscalar() for l in losses])
print('Epoch %d: Loss: %.3f, Time %.1f sec' % (epoch, train_loss/len(train_data)/batch_size, time()-tic))
# Device configuration
device = mx.cpu() if len(mx.test_utils.list_gpus()) == 0 else mx.gpu()
print('current ctx =>', device)
# Hyper-parameters
num_epochs = 5
num_classes = 10
batch_size = 100
learning_rate = 0.001
# FashionMNIST (images and labels)
transformer = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(0.13, 0.31)])
train_dataset = datasets.FashionMNIST(root='../../data', train=True)
train_dataset = train_dataset.transform_first(transformer)
test_dataset = datasets.FashionMNIST(root='../../data', train=False)
test_dataset = test_dataset.transform_first(transformer)
# DataLoader (input pipeline)
train_loader = gluon.data.DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)
test_loader = gluon.data.DataLoader(dataset=test_dataset,
batch_size=batch_size,
shuffle=False)
net.load_params('net.params', ctx=gpu(0))
x = nd.random.uniform(shape=(1, 1, 28, 28), ctx=gpu(0))
print(net(x))
# advanced multi-gpu training
batch_size = 256
transformer = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(0.13, 0.31)])
train_data = gluon.data.DataLoader(
datasets.FashionMNIST(train=True).transform_first(transformer),
batch_size,
shuffle=True,
num_workers=4)
valid_data = gluon.data.DataLoader(
datasets.FashionMNIST(train=False).transform_first(transformer),
batch_size,
shuffle=False,
num_workers=4)
# devices = [gpu(0), gpu(1)]
devices = [gpu(x) for x in range(gpu_count)]
net.collect_params().initialize(force_reinit=True, ctx=devices)
from mxnet import nd, gluon, init, autograd
from mxnet.gluon import nn
from mxnet.gluon.data.vision import datasets, transforms
from IPython import display
import matplotlib.pyplot as plt
import time
# Get data
mnist_train = datasets.FashionMNIST(
root='./data',
train=True)
X, y = mnist_train[0]
print(('X shape: ', X.shape, 'X dtype: ', X.dtype, 'y: ', y))
text_labels = [
't-shirts', 'trouser', 'pullover', 'dress', 'coat',
'sandal', 'shirt', 'sneaker', 'bag', 'ankle boot']
X, y = mnist_train[0:10]
# plot images
display.set_matplotlib_formats('svg')
_, figs = plt.subplots(1, X.shape[0], figsize=(15, 15))
for f, x, yi in zip(figs, X, y):
# 3D -> 2D by removing the last channel dim
f.imshow(x.reshape((28, 28)).asnumpy())
ax = f.axes
ax.set_title(text_labels[int(yi)])
ax.title.set_fontsize(14)
