def main():
HERE = os.path.dirname(__file__)
# Import MNIST data
sys.path.append(
os.path.realpath(os.path.join(HERE, '..', '..', 'vision', 'mnist')))
from mnist_data import data_iterator_mnist
# Create binary output folder
path_bin = os.path.join(HERE, "mnist_images")
if not os.path.isdir(path_bin):
os.makedirs(path_bin)
# Get MNIST testing images.
images, labels = data_iterator_mnist(10000, train=False,
shuffle=True).next()
# Dump image binary files with row-major order.
for i in range(10):
outfile = os.path.join(path_bin, "{}.pgm".format(i))
print("Generator a binary file of number {} to {}".format(i, outfile))
ind = np.where(labels == i)[0][0]
image = images[ind].copy(order='C')
with open(outfile, 'w') as fd:
print('P5', file=fd)
print('# Created by nnabla mnist_runtime example.', file=fd)
print('28 28', file=fd)
print('255', file=fd)
image.tofile(fd)
def main():
HERE = os.path.dirname(__file__)
# Import MNIST data
sys.path.append(
os.path.realpath(os.path.join(HERE, '..', '..', 'vision', 'mnist')))
from mnist_data import data_iterator_mnist
# Create binary output folder
path_bin = os.path.join(HERE, "mnist_images")
if not os.path.isdir(path_bin):
os.makedirs(path_bin)
# Get MNIST testing images.
images, labels = data_iterator_mnist(
10000, train=False, shuffle=True).next()
# Dump image binary files with row-major order.
for i in range(10):
outfile = os.path.join(path_bin, "{}.pgm".format(i))
print("Generator a binary file of number {} to {}".format(i, outfile))
ind = np.where(labels == i)[0][0]
image = images[ind].copy(order='C')
with open(outfile, 'w') as fd:
print('P5', file=fd)
print('# Created by nnabla mnist_runtime example.', file=fd)
print('28 28', file=fd)
print('255', file=fd)
image.tofile(fd)
def test():
print("Evaluate the trained model with full MNIST test set")
args = get_args()
# Create CNN network for both training and testing.
mnist_cnn_prediction = mnist_lenet_prediction
if args.net == 'resnet':
mnist_cnn_prediction = mnist_resnet_prediction
args.batch_size = 100
tdata = data_iterator_mnist(args.batch_size, False)
timage = nn.Variable([args.batch_size, 1, 28, 28])
tlabel = nn.Variable([args.batch_size, 1])
parameter_file = os.path.join(
args.model_save_path,
'{}_params_{:06}.h5'.format(args.net, args.max_iter))
nn.load_parameters(parameter_file)
# Create inference graph
tpred = mnist_cnn_prediction(timage, test=True)
num_test_iter = int((tdata.size + args.batch_size - 1) / args.batch_size)
te = 0.0
for j in range(num_test_iter):
timage.d, tlabel.d = tdata.next()
tpred.forward(clear_buffer=True)
te += categorical_error(tpred.d, tlabel.d)
te_avg = te / num_test_iter
print("MNIST test accuracy", 1 - te_avg)
def main():
# Context
ctx = get_extension_context("cudnn", device_id="0")
nn.set_default_context(ctx)
nn.auto_forward(False)
# Inputs
b, c, h, w = 64, 1, 28, 28
x = nn.Variable([b, c, h, w])
t = nn.Variable([b, 1])
vx = nn.Variable([b, c, h, w])
vt = nn.Variable([b, 1])
# Model
model = Model()
pred = model(x)
loss = F.softmax_cross_entropy(pred, t)
vpred = model(vx, test=True)
verror = F.top_n_error(vpred, vt)
# Solver
solver = S.Adam()
solver.set_parameters(model.get_parameters(grad_only=True))
# Data Iterator
tdi = data_iterator_mnist(b, train=True)
vdi = data_iterator_mnist(b, train=False)
# Monitor
monitor = Monitor("tmp.monitor")
monitor_loss = MonitorSeries("Training loss", monitor, interval=10)
monitor_verr = MonitorSeries("Test error", monitor, interval=1)
# Training loop
for e in range(1):
for j in range(tdi.size // b):
i = e * tdi.size // b + j
x.d, t.d = tdi.next()
solver.zero_grad()
loss.forward(clear_no_need_grad=True)
loss.backward(clear_buffer=True)
solver.update()
monitor_loss.add(i, loss.d)
error = 0.0
for _ in range(vdi.size // b):
vx.d, vt.d = vdi.next()
verror.forward(clear_buffer=True)
error += verror.d
error /= vdi.size // b
monitor_verr.add(i, error)
def mnist_iterator(config, comm, train=True):
data_iterator_ = data_iterator_mnist(batch_size=config['train']['batch_size'], train=train, rng=np.random.RandomState(config['model']['rng']),
with_memory_cache=config['dataset']['with_memory_cache'], with_file_cache=config['dataset']['with_file_cache'])
if comm.n_procs > 1:
data_iterator_ = data_iterator_.slice(
rng=None, num_of_slices=comm.n_procs, slice_pos=comm.rank)
return data_iterator_
def visualize(args):
"""
Visualizing embedded digits onto 2D space.
"""
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
batch_size = 500
# Create default context.
ctx = nn.Context(backend="cpu|cuda",
compute_backend="default|cudnn",
array_class="CudaArray",
device_id="{}".format(args.device_id))
# Load parameters
nn.load_parameters(
os.path.join(args.model_save_path, 'params_%06d.h5' % args.max_iter))
# Create embedder network
image = nn.Variable([batch_size, 1, 28, 28])
feature = mnist_lenet_feature(image, test=False)
# Process all images
features = []
labels = []
# Prepare MNIST data iterator
rng = np.random.RandomState(313)
data = data_iterator_mnist(batch_size, train=False, shuffle=True, rng=rng)
for i in range(10000 // batch_size):
image_data, label_data = data.next()
image.d = image_data / 255.
feature.forward(clear_buffer=True)
features.append(feature.d.copy())
labels.append(label_data.copy())
features = np.vstack(features)
labels = np.vstack(labels)
# Visualize
f = plt.figure(figsize=(16, 9))
for i in range(10):
c = plt.cm.Set1(i / 10.)
plt.plot(features[labels.flat == i, 0].flatten(),
features[labels.flat == i, 1].flatten(),
'.',
c=c)
plt.legend(map(str, range(10)))
plt.grid()
plt.savefig(os.path.join(args.monitor_path, "embed.png"))
def visualize(args):
"""
Visualizing embedded digits onto 2D space.
"""
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
batch_size = 500
# Create default context.
ctx = nn.Context(backend="cpu|cuda",
compute_backend="default|cudnn",
array_class="CudaArray",
device_id="{}".format(args.device_id))
# Load parameters
nn.load_parameters(os.path.join(args.model_save_path,
'params_%06d.h5' % args.max_iter))
# Create embedder network
image = nn.Variable([batch_size, 1, 28, 28])
feature = mnist_lenet_feature(image, test=False)
# Process all images
features = []
labels = []
# Prepare MNIST data iterator
rng = np.random.RandomState(313)
data = data_iterator_mnist(batch_size, train=False, shuffle=True, rng=rng)
for i in range(10000 // batch_size):
image_data, label_data = data.next()
image.d = image_data / 255.
feature.forward(clear_buffer=True)
features.append(feature.d.copy())
labels.append(label_data.copy())
features = np.vstack(features)
labels = np.vstack(labels)
# Visualize
f = plt.figure(figsize=(16, 9))
for i in range(10):
c = plt.cm.Set1(i / 10.)
plt.plot(features[labels.flat == i, 0].flatten(), features[
labels.flat == i, 1].flatten(), '.', c=c)
plt.legend(map(str, range(10)))
plt.grid()
plt.savefig(os.path.join(args.monitor_path, "embed.png"))
def train():
args = get_args()
# Get context.
from nnabla.contrib.context import extension_context
extension_module = args.context
if args.context is None:
extension_module = 'cpu'
logger.info("Running in %s" % extension_module)
ctx = extension_context(extension_module, device_id=args.device_id)
nn.set_default_context(ctx)
# Create CNN network for both training and testing.
mnist_cnn_prediction = mnist_lenet_prediction
# TRAIN
reference = "reference"
# Create input variables.
image = nn.Variable([args.batch_size, 1, 28, 28])
label = nn.Variable([args.batch_size, 1])
# Create `reference` prediction graph.
pred = mnist_cnn_prediction(image, scope=reference, test=False)
pred.persistent = True
# Create loss function.
loss = F.mean(F.softmax_cross_entropy(pred, label))
# TEST
# Create input variables.
vimage = nn.Variable([args.batch_size, 1, 28, 28])
vlabel = nn.Variable([args.batch_size, 1])
# Create reference predition graph.
vpred = mnist_cnn_prediction(vimage, scope=reference, test=True)
# Create Solver.
solver = S.Adam(args.learning_rate)
solver.set_parameters(nn.get_parameters())
# Create monitor.
from nnabla.monitor import Monitor, MonitorSeries, MonitorTimeElapsed
monitor = Monitor(args.monitor_path)
monitor_loss = MonitorSeries("Training loss", monitor, interval=10)
monitor_err = MonitorSeries("Training error", monitor, interval=10)
monitor_time = MonitorTimeElapsed("Training time", monitor, interval=100)
monitor_verr = MonitorSeries("Test error", monitor, interval=10)
# Initialize DataIterator for MNIST.
data = data_iterator_mnist(args.batch_size, True)
vdata = data_iterator_mnist(args.batch_size, False)
best_ve = 1.0
ve = 1.0
# Training loop.
for i in range(args.max_iter):
if i % args.val_interval == 0:
# Validation
ve = 0.0
for j in range(args.val_iter):
vimage.d, vlabel.d = vdata.next()
vpred.forward(clear_buffer=True)
ve += categorical_error(vpred.d, vlabel.d)
ve /= args.val_iter
monitor_verr.add(i, ve)
if ve < best_ve:
nn.save_parameters(
os.path.join(args.model_save_path, 'params_%06d.h5' % i))
best_ve = ve
# Training forward
image.d, label.d = data.next()
solver.zero_grad()
loss.forward(clear_no_need_grad=True)
loss.backward(clear_buffer=True)
solver.weight_decay(args.weight_decay)
solver.update()
e = categorical_error(pred.d, label.d)
monitor_loss.add(i, loss.d.copy())
monitor_err.add(i, e)
monitor_time.add(i)
ve = 0.0
for j in range(args.val_iter):
vimage.d, vlabel.d = vdata.next()
vpred.forward(clear_buffer=True)
ve += categorical_error(vpred.d, vlabel.d)
monitor_verr.add(i, ve / args.val_iter)
parameter_file = os.path.join(args.model_save_path,
'params_{:06}.h5'.format(args.max_iter))
nn.save_parameters(parameter_file)
def main():
"""
Main script.
Steps:
* Setup calculation environment
* Initialize data iterator.
* Create Networks
* Create Solver.
* Training Loop.
* Training
* Test
* Save
"""
# Set args
args = get_args(monitor_path='tmp.monitor.vae',
max_iter=60000,
model_save_path=None,
learning_rate=3e-4,
batch_size=100,
weight_decay=0)
# Get context.
from nnabla.ext_utils import get_extension_context
logger.info("Running in %s" % args.context)
ctx = get_extension_context(args.context,
device_id=args.device_id,
type_config=args.type_config)
nn.set_default_context(ctx)
# Initialize data provider
di_l = data_iterator_mnist(args.batch_size, True)
di_t = data_iterator_mnist(args.batch_size, False)
# Network
shape_x = (1, 28, 28)
shape_z = (50, )
x = nn.Variable((args.batch_size, ) + shape_x)
loss_l = vae(x, shape_z, test=False)
loss_t = vae(x, shape_z, test=True)
# Create solver
solver = S.Adam(args.learning_rate)
solver.set_parameters(nn.get_parameters())
# Monitors for training and validation
monitor = M.Monitor(args.model_save_path)
monitor_training_loss = M.MonitorSeries("Training loss",
monitor,
interval=600)
monitor_test_loss = M.MonitorSeries("Test loss", monitor, interval=600)
monitor_time = M.MonitorTimeElapsed("Elapsed time", monitor, interval=600)
# Training Loop.
for i in range(args.max_iter):
# Initialize gradients
solver.zero_grad()
# Forward, backward and update
x.d, _ = di_l.next()
loss_l.forward(clear_no_need_grad=True)
loss_l.backward(clear_buffer=True)
solver.weight_decay(args.weight_decay)
solver.update()
# Forward for test
x.d, _ = di_t.next()
loss_t.forward(clear_no_need_grad=True)
# Monitor for logging
monitor_training_loss.add(i, loss_l.d.copy())
monitor_test_loss.add(i, loss_t.d.copy())
monitor_time.add(i)
# Save the model
nn.save_parameters(
os.path.join(args.model_save_path, 'params_%06d.h5' % args.max_iter))
def siamese_data_iterator(batch_size, train, rng=None):
itr0 = data_iterator_mnist(batch_size, train=train, shuffle=True, rng=rng)
itr1 = data_iterator_mnist(batch_size, train=train, shuffle=True, rng=rng)
return MnistSiameseDataIterator(itr0, itr1)
def siamese_data_iterator(batch_size, train, rng=None):
itr0 = data_iterator_mnist(batch_size, train=train, rng=rng, shuffle=True)
itr1 = data_iterator_mnist(batch_size, train=train, rng=rng, shuffle=True)
return MnistSiameseDataIterator(itr0, itr1)
def distil():
args = get_args()
# Get context.
from nnabla.contrib.context import extension_context
extension_module = args.context
if args.context is None:
extension_module = 'cpu'
logger.info("Running in %s" % extension_module)
ctx = extension_context(extension_module, device_id=args.device_id)
nn.set_default_context(ctx)
# Create CNN network for both training and testing.
mnist_cnn_prediction = mnist_resnet_prediction
# TRAIN
teacher = "teacher"
student = "student"
# Create input variables.
image = nn.Variable([args.batch_size, 1, 28, 28])
image.persistent = True # not clear the intermediate buffer re-used
label = nn.Variable([args.batch_size, 1])
label.persistent = True # not clear the intermediate buffer re-used
# Create `teacher` and "student" prediction graph.
model_load_path = args.model_load_path
nn.load_parameters(model_load_path)
pred_label = mnist_cnn_prediction(image, net=teacher, maps=64, test=False)
pred_label.need_grad = False # no need backward through teacher graph
pred = mnist_cnn_prediction(image, net=student, maps=32, test=False)
pred.persistent = True # not clear the intermediate buffer used
loss_ce = F.mean(F.softmax_cross_entropy(pred, label))
loss_kl = kl_divergence(pred, pred_label)
loss = args.weight_ce * loss_ce + args.weight_kl * loss_kl
# TEST
# Create input variables.
vimage = nn.Variable([args.batch_size, 1, 28, 28])
vlabel = nn.Variable([args.batch_size, 1])
# Create teacher predition graph.
vpred = mnist_cnn_prediction(vimage, net=student, maps=32, test=True)
# Create Solver.
solver = S.Adam(args.learning_rate)
with nn.parameter_scope(student):
solver.set_parameters(nn.get_parameters())
# Create monitor.
from nnabla.monitor import Monitor, MonitorSeries, MonitorTimeElapsed
monitor = Monitor(args.monitor_path)
monitor_loss = MonitorSeries("Training loss", monitor, interval=10)
monitor_err = MonitorSeries("Training error", monitor, interval=10)
monitor_time = MonitorTimeElapsed("Training time", monitor, interval=100)
monitor_verr = MonitorSeries("Test error", monitor, interval=10)
# Initialize DataIterator for MNIST.
data = data_iterator_mnist(args.batch_size, True)
vdata = data_iterator_mnist(args.batch_size, False)
best_ve = 1.0
# Training loop.
for i in range(args.max_iter):
if i % args.val_interval == 0:
# Validation
ve = 0.0
for j in range(args.val_iter):
vimage.d, vlabel.d = vdata.next()
vpred.forward(clear_buffer=True)
ve += categorical_error(vpred.d, vlabel.d)
monitor_verr.add(i, ve / args.val_iter)
if ve < best_ve:
nn.save_parameters(os.path.join(
args.model_save_path, 'params_%06d.h5' % i))
best_ve = ve
# Training forward
image.d, label.d = data.next()
solver.zero_grad()
loss.forward(clear_no_need_grad=True)
loss.backward(clear_buffer=True)
solver.weight_decay(args.weight_decay)
solver.update()
e = categorical_error(pred.d, label.d)
monitor_loss.add(i, loss.d.copy())
monitor_err.add(i, e)
monitor_time.add(i)
ve = 0.0
for j in range(args.val_iter):
vimage.d, vlabel.d = vdata.next()
vpred.forward(clear_buffer=True)
ve += categorical_error(vpred.d, vlabel.d)
monitor_verr.add(i, ve / args.val_iter)
parameter_file = os.path.join(
args.model_save_path, 'params_{:06}.h5'.format(args.max_iter))
nn.save_parameters(parameter_file)
logger.debug('output: {}'.format(args.output))
logger.debug('normalize: {}'.format(args.normalize))
logger.debug('max_epoch: {}'.format(args.max_epoch))
logger.debug('wait: {}'.format(args.wait))
nnabla_config.set('DATA_ITERATOR', 'data_source_file_cache_size',
'{}'.format(args.cache_size))
nnabla_config.set('DATA_ITERATOR', 'data_source_buffer_max_size',
'{}'.format(args.memory_size))
if args.uri == 'MNIST_TRAIN':
sys.path.append(
os.path.join(os.path.dirname(os.path.abspath(__file__)), '..',
'vision', 'mnist'))
from mnist_data import data_iterator_mnist
with data_iterator_mnist(args.batch_size, True, None, args.shuffle,
args.memory_cache, args.file_cache) as di:
test_data_iterator(di, args)
elif args.uri == 'MNIST_TEST':
sys.path.append(
os.path.join(os.path.dirname(os.path.abspath(__file__)), '..',
'vision', 'mnist'))
from mnist_data import data_iterator_mnist
with data_iterator_mnist(args.batch_size, False, None, args.shuffle,
args.memory_cache, args.file_cache) as di:
test_data_iterator(di, args)
elif args.uri == 'TINY_IMAGENET_TRAIN':
sys.path.append(
os.path.join(os.path.dirname(os.path.abspath(__file__)), '..',
'vision', 'imagenet'))
from tiny_imagenet_data import data_iterator_tiny_imagenet
with data_iterator_tiny_imagenet(args.batch_size, 'train') as di:
def train(max_iter=24000):
shape_x = (1, 28, 28)
n_h = args.n_units
n_y = args.n_class
# Load MNIST Dataset
from mnist_data import load_mnist, data_iterator_mnist
images, labels = load_mnist(train=True)
rng = np.random.RandomState(706)
inds = rng.permutation(len(images))
def feed_labeled(i):
j = inds[i]
return images[j], labels[j]
def feed_unlabeled(i):
j = inds[i]
return images[j], labels[j]
di_l = I.data_iterator_simple(
feed_labeled,
args.n_labeled,
args.batchsize_l,
shuffle=True,
rng=rng,
with_file_cache=False,
)
di_u = I.data_iterator_simple(
feed_unlabeled,
args.n_train,
args.batchsize_u,
shuffle=True,
rng=rng,
with_file_cache=False,
)
di_v = data_iterator_mnist(args.batchsize_v, train=False)
# Create networks
# feed-forward-net building function
def forward(x, test=False):
return I.mlp_net(x, n_h, n_y, test)
# Net for learning labeled data
xl = nn.Variable((args.batchsize_l,) + shape_x, need_grad=False)
yl = forward(xl, test=False)
tl = nn.Variable((args.batchsize_l, 1), need_grad=False)
loss_l = F.mean(F.softmax_cross_entropy(yl, tl))
# Net for learning unlabeled data
xu = nn.Variable((args.batchsize_u,) + shape_x, need_grad=False)
yu = forward(xu, test=False)
y1 = yu.get_unlinked_variable()
y1.need_grad = False
noise = nn.Variable((args.batchsize_u,) + shape_x, need_grad=True)
r = noise / (F.sum(noise ** 2, [1, 2, 3], keepdims=True)) ** 0.5
r.persistent = True
y2 = forward(xu + args.xi_for_vat * r, test=False)
y3 = forward(xu + args.eps_for_vat * r, test=False)
loss_k = F.mean(I.distance(y1, y2))
loss_u = F.mean(I.distance(y1, y3))
# Net for evaluating validation data
xv = nn.Variable((args.batchsize_v,) + shape_x, need_grad=False)
hv = forward(xv, test=True)
tv = nn.Variable((args.batchsize_v, 1), need_grad=False)
err = F.mean(F.top_n_error(hv, tv, n=1))
# Create solver
solver = S.Adam(args.learning_rate)
solver.set_parameters(nn.get_parameters())
# Monitor training and validation stats.
path = cache_dir(os.path.join(I.name, "monitor"))
monitor = M.Monitor(path)
monitor_verr = M.MonitorSeries("val_error", monitor, interval=240)
monitor_time = M.MonitorTimeElapsed("time", monitor, interval=240)
# Training Loop.
for i in range(max_iter):
# Validation Test
if i % args.val_interval == 0:
valid_error = I.calc_validation_error(di_v, xv, tv, err, args.val_iter)
monitor_verr.add(i, valid_error)
# forward, backward and update
xl.d, tl.d = di_l.next()
xl.d = xl.d / 255
solver.zero_grad()
loss_l.forward(clear_no_need_grad=True)
loss_l.backward(clear_buffer=True)
solver.weight_decay(args.weight_decay)
solver.update()
# Calculate y without noise, only once.
xu.d, _ = di_u.next()
xu.d = xu.d / 255
yu.forward(clear_buffer=True)
# Do power method iteration
noise.d = np.random.normal(size=xu.shape).astype(np.float32)
for k in range(args.n_iter_for_power_method):
r.grad.zero()
loss_k.forward(clear_no_need_grad=True)
loss_k.backward(clear_buffer=True)
noise.data.copy_from(r.grad)
# forward, backward and update
solver.zero_grad()
loss_u.forward(clear_no_need_grad=True)
loss_u.backward(clear_buffer=True)
solver.weight_decay(args.weight_decay)
solver.update()
if i % args.iter_per_epoch == 0:
solver.set_learning_rate(solver.learning_rate() * args.learning_rate_decay)
monitor_time.add(i)
# Evaluate the final model by the error rate with validation dataset
valid_error = I.calc_validation_error(di_v, xv, tv, err, args.val_iter)
monitor_verr.add(i, valid_error)
monitor_time.add(i)
return path
def train(args):
"""
Main script.
"""
# Get context.
from nnabla.contrib.context import extension_context
extension_module = args.context
if args.context is None:
extension_module = 'cpu'
logger.info("Running in %s" % extension_module)
ctx = extension_context(extension_module, device_id=args.device_id)
nn.set_default_context(ctx)
# Create CNN network for both training and testing.
# TRAIN
# Fake path
z = nn.Variable([args.batch_size, 100, 1, 1])
zeros = nn.Variable([args.batch_size, 1])
zeros.data.zero()
ones = nn.Variable([args.batch_size, 1])
ones.data.fill(1)
fake = generator(z)
fake.persistent = True # Not to clear at backward
pred_fake = discriminator(fake)
loss_gen = F.mean(F.sigmoid_cross_entropy(pred_fake, ones))
loss_dis = F.mean(F.sigmoid_cross_entropy(pred_fake, zeros))
# Real path
x = nn.Variable([args.batch_size, 1, 28, 28])
pred_real = discriminator(x)
loss_dis += F.mean(F.sigmoid_cross_entropy(pred_real, ones))
# Create Solver.
solver_gen = S.Adam(args.learning_rate, beta1=0.5)
solver_dis = S.Adam(args.learning_rate, beta1=0.5)
with nn.parameter_scope("gen"):
solver_gen.set_parameters(nn.get_parameters())
with nn.parameter_scope("dis"):
solver_dis.set_parameters(nn.get_parameters())
# Create monitor.
import nnabla.monitor as M
monitor = M.Monitor(args.monitor_path)
monitor_loss_gen = M.MonitorSeries("Generator loss", monitor, interval=10)
monitor_loss_dis = M.MonitorSeries("Discriminator loss",
monitor,
interval=10)
monitor_time = M.MonitorTimeElapsed("Time", monitor, interval=100)
monitor_fake = M.MonitorImageTile("Fake images",
monitor,
normalize_method=lambda x: x + 1 / 2.)
data = data_iterator_mnist(args.batch_size, True)
# Training loop.
for i in range(args.max_iter):
if i % args.model_save_interval == 0:
with nn.parameter_scope("gen"):
nn.save_parameters(
os.path.join(args.model_save_path,
"generator_param_%06d.h5" % i))
with nn.parameter_scope("dis"):
nn.save_parameters(
os.path.join(args.model_save_path,
"discriminator_param_%06d.h5" % i))
# Training forward
image, _ = data.next()
x.d = image / 255. - 0.5 # [0, 255] to [-1, 1]
z.d = np.random.randn(*z.shape)
# Generator update.
solver_gen.zero_grad()
loss_gen.forward(clear_no_need_grad=True)
loss_gen.backward(clear_buffer=True)
solver_gen.weight_decay(args.weight_decay)
solver_gen.update()
monitor_fake.add(i, fake)
monitor_loss_gen.add(i, loss_gen.d.copy())
# Discriminator update.
solver_dis.zero_grad()
loss_dis.forward(clear_no_need_grad=True)
loss_dis.backward(clear_buffer=True)
solver_dis.weight_decay(args.weight_decay)
solver_dis.update()
monitor_loss_dis.add(i, loss_dis.d.copy())
monitor_time.add(i)
with nn.parameter_scope("gen"):
nn.save_parameters(
os.path.join(args.model_save_path,
"generator_param_%06d.h5" % args.max_iter))
with nn.parameter_scope("dis"):
nn.save_parameters(
os.path.join(args.model_save_path,
"discriminator_param_%06d.h5" % args.max_iter))
def train():
"""
Main script.
Steps:
* Parse command line arguments.
* Specify a context for computation.
* Initialize DataIterator for MNIST.
* Construct a computation graph for training and validation.
* Initialize a solver and set parameter variables to it.
* Create monitor instances for saving and displaying training stats.
* Training loop
* Computate error rate for validation data (periodically)
* Get a next minibatch.
* Execute forwardprop on the training graph.
* Compute training error
* Set parameter gradients zero
* Execute backprop.
* Solver updates parameters by using gradients computed by backprop.
"""
args = get_args()
# Get context.
from nnabla.contrib.context import extension_context
extension_module = args.context
if args.context is None:
extension_module = 'cpu'
logger.info("Running in %s" % extension_module)
ctx = extension_context(extension_module, device_id=args.device_id)
nn.set_default_context(ctx)
# Create CNN network for both training and testing.
mnist_cnn_prediction = mnist_lenet_prediction
if args.net == 'resnet':
mnist_cnn_prediction = mnist_resnet_prediction
# TRAIN
# Create input variables.
image = nn.Variable([args.batch_size, 1, 28, 28])
label = nn.Variable([args.batch_size, 1])
# Create prediction graph.
pred = mnist_cnn_prediction(image, test=False)
pred.persistent = True
# Create loss function.
loss = F.mean(F.softmax_cross_entropy(pred, label))
# TEST
# Create input variables.
vimage = nn.Variable([args.batch_size, 1, 28, 28])
vlabel = nn.Variable([args.batch_size, 1])
# Create predition graph.
vpred = mnist_cnn_prediction(vimage, test=True)
# Create Solver.
solver = S.Adam(args.learning_rate)
solver.set_parameters(nn.get_parameters())
# Create monitor.
from nnabla.monitor import Monitor, MonitorSeries, MonitorTimeElapsed
monitor = Monitor(args.monitor_path)
monitor_loss = MonitorSeries("Training loss", monitor, interval=10)
monitor_err = MonitorSeries("Training error", monitor, interval=10)
monitor_time = MonitorTimeElapsed("Training time", monitor, interval=100)
monitor_verr = MonitorSeries("Test error", monitor, interval=10)
# Initialize DataIterator for MNIST.
data = data_iterator_mnist(args.batch_size, True)
vdata = data_iterator_mnist(args.batch_size, False)
# Training loop.
for i in range(args.max_iter):
if i % args.val_interval == 0:
# Validation
ve = 0.0
for j in range(args.val_iter):
vimage.d, vlabel.d = vdata.next()
vpred.forward(clear_buffer=True)
ve += categorical_error(vpred.d, vlabel.d)
monitor_verr.add(i, ve / args.val_iter)
if i % args.model_save_interval == 0:
nn.save_parameters(os.path.join(
args.model_save_path, 'params_%06d.h5' % i))
# Training forward
image.d, label.d = data.next()
solver.zero_grad()
loss.forward(clear_no_need_grad=True)
loss.backward(clear_buffer=True)
solver.weight_decay(args.weight_decay)
solver.update()
e = categorical_error(pred.d, label.d)
monitor_loss.add(i, loss.d.copy())
monitor_err.add(i, e)
monitor_time.add(i)
ve = 0.0
for j in range(args.val_iter):
vimage.d, vlabel.d = vdata.next()
vpred.forward(clear_buffer=True)
ve += categorical_error(vpred.d, vlabel.d)
monitor_verr.add(i, ve / args.val_iter)
parameter_file = os.path.join(
args.model_save_path, '{}_params_{:06}.h5'.format(args.net, args.max_iter))
nn.save_parameters(parameter_file)
def train():
args = get_args()
# Get context.
from nnabla.contrib.context import extension_context
extension_module = args.context
if args.context is None:
extension_module = 'cpu'
logger.info("Running in %s" % extension_module)
ctx = extension_context(extension_module, device_id=args.device_id)
nn.set_default_context(ctx)
# Train
image = nn.Variable([args.batch_size, 1, 28, 28])
label = nn.Variable([args.batch_size, 1])
h_d, h_copy, pred, g_pred, g_label = cnn_dni(image, y=label)
loss_ce = ce_loss(pred, label) # loss of a problem at hand
loss_se = se_loss(g_pred, g_label) # gradient synthesizer loss
# Test
vimage = nn.Variable([args.batch_size, 1, 28, 28])
vlabel = nn.Variable([args.batch_size, 1])
vpred = cnn(vimage, test=True)
# Solver
solver = S.Adam(args.learning_rate)
with nn.parameter_scope("ref"):
solver.set_parameters(nn.get_parameters())
solver_gs = S.Adam(args.learning_rate)
with nn.parameter_scope("gs"):
solver_gs.set_parameters(nn.get_parameters())
# Monitor
from nnabla.monitor import Monitor, MonitorSeries, MonitorTimeElapsed
monitor = Monitor(args.monitor_path)
monitor_loss = MonitorSeries("Training loss", monitor, interval=10)
monitor_err = MonitorSeries("Training error", monitor, interval=10)
monitor_time = MonitorTimeElapsed("Training time", monitor, interval=100)
monitor_verr = MonitorSeries("Test error", monitor, interval=10)
# DataIterator
data = data_iterator_mnist(args.batch_size, True)
vdata = data_iterator_mnist(args.batch_size, False)
# Training loop
for i in range(args.max_iter):
if i % args.val_interval == 0:
# Validation
ve = 0.0
for j in range(args.val_iter):
vimage.d, vlabel.d = vdata.next()
vpred.forward(clear_buffer=False)
ve += categorical_error(vpred.d, vlabel.d)
monitor_verr.add(i, ve / args.val_iter)
if i % args.model_save_interval == 0:
nn.save_parameters(
os.path.join(args.model_save_path, 'params_%06d.h5' % i))
# Training
image.d, label.d = data.next()
solver.zero_grad()
solver_gs.zero_grad()
## forward
h_d.forward(clear_no_need_grad=False)
loss_ce.forward(clear_no_need_grad=False)
loss_se.forward(clear_no_need_grad=False)
## backward
loss_ce.backward(clear_buffer=False)
h_d.backward(clear_buffer=False)
loss_se.backward(clear_buffer=False)
## update
solver.weight_decay(args.weight_decay)
solver.update()
solver_gs.weight_decay(args.weight_decay)
solver_gs.update()
## monitor
e = categorical_error(pred.d, label.d)
monitor_loss.add(i, loss_ce.d.copy())
monitor_err.add(i, e)
monitor_time.add(i)
nn.save_parameters(
os.path.join(args.model_save_path, 'params_%06d.h5' % args.max_iter))
def train(args):
"""
Main script.
"""
# Get context.
from nnabla.ext_utils import get_extension_context
logger.info("Running in %s" % args.context)
ctx = get_extension_context(args.context,
device_id=args.device_id,
type_config=args.type_config)
nn.set_default_context(ctx)
# Create CNN network for both training and testing.
# TRAIN
# Fake path
z = nn.Variable([args.batch_size, 100, 1, 1])
fake = generator(z)
fake.persistent = True # Not to clear at backward
pred_fake = discriminator(fake)
loss_gen = F.mean(
F.sigmoid_cross_entropy(pred_fake, F.constant(1, pred_fake.shape)))
fake_dis = fake.get_unlinked_variable(need_grad=True)
fake_dis.need_grad = True # TODO: Workaround until v1.0.2
pred_fake_dis = discriminator(fake_dis)
loss_dis = F.mean(
F.sigmoid_cross_entropy(pred_fake_dis,
F.constant(0, pred_fake_dis.shape)))
# Real path
x = nn.Variable([args.batch_size, 1, 28, 28])
pred_real = discriminator(x)
loss_dis += F.mean(
F.sigmoid_cross_entropy(pred_real, F.constant(1, pred_real.shape)))
# Create Solver.
solver_gen = S.Adam(args.learning_rate, beta1=0.5)
solver_dis = S.Adam(args.learning_rate, beta1=0.5)
with nn.parameter_scope("gen"):
solver_gen.set_parameters(nn.get_parameters())
with nn.parameter_scope("dis"):
solver_dis.set_parameters(nn.get_parameters())
start_point = 0
if args.checkpoint is not None:
# load weights and solver state info from specified checkpoint files.
start_point = load_checkpoint(args.checkpoint, {
"gen": solver_gen,
"dis": solver_dis
})
# Create monitor.
import nnabla.monitor as M
monitor = M.Monitor(args.monitor_path)
monitor_loss_gen = M.MonitorSeries("Generator loss", monitor, interval=10)
monitor_loss_dis = M.MonitorSeries("Discriminator loss",
monitor,
interval=10)
monitor_time = M.MonitorTimeElapsed("Time", monitor, interval=100)
monitor_fake = M.MonitorImageTile("Fake images",
monitor,
normalize_method=lambda x: (x + 1) / 2.)
data = data_iterator_mnist(args.batch_size, True)
# Save_nnp
contents = save_nnp({'x': z}, {'y': fake}, args.batch_size)
save.save(
os.path.join(args.model_save_path, 'Generator_result_epoch0.nnp'),
contents)
contents = save_nnp({'x': x}, {'y': pred_real}, args.batch_size)
save.save(
os.path.join(args.model_save_path, 'Discriminator_result_epoch0.nnp'),
contents)
# Training loop.
for i in range(start_point, args.max_iter):
if i % args.model_save_interval == 0:
save_checkpoint(args.model_save_path, i, {
"gen": solver_gen,
"dis": solver_dis
})
# Training forward
image, _ = data.next()
x.d = image / 255. - 0.5 # [0, 255] to [-1, 1]
z.d = np.random.randn(*z.shape)
# Generator update.
solver_gen.zero_grad()
loss_gen.forward(clear_no_need_grad=True)
loss_gen.backward(clear_buffer=True)
solver_gen.weight_decay(args.weight_decay)
solver_gen.update()
monitor_fake.add(i, fake)
monitor_loss_gen.add(i, loss_gen.d.copy())
# Discriminator update.
solver_dis.zero_grad()
loss_dis.forward(clear_no_need_grad=True)
loss_dis.backward(clear_buffer=True)
solver_dis.weight_decay(args.weight_decay)
solver_dis.update()
monitor_loss_dis.add(i, loss_dis.d.copy())
monitor_time.add(i)
with nn.parameter_scope("gen"):
nn.save_parameters(
os.path.join(args.model_save_path, "generator_param_%06d.h5" % i))
with nn.parameter_scope("dis"):
nn.save_parameters(
os.path.join(args.model_save_path,
"discriminator_param_%06d.h5" % i))
# Save_nnp
contents = save_nnp({'x': z}, {'y': fake}, args.batch_size)
save.save(os.path.join(args.model_save_path, 'Generator_result.nnp'),
contents)
contents = save_nnp({'x': x}, {'y': pred_real}, args.batch_size)
save.save(os.path.join(args.model_save_path, 'Discriminator_result.nnp'),
contents)
def train(args):
"""
Main script.
"""
# Get context.
from nnabla.contrib.context import extension_context
extension_module = args.context
if args.context is None:
extension_module = 'cpu'
logger.info("Running in %s" % extension_module)
ctx = extension_context(extension_module, device_id=args.device_id)
nn.set_default_context(ctx)
# Create CNN network for both training and testing.
# TRAIN
# Fake path
z = nn.Variable([args.batch_size, 100, 1, 1])
fake = generator(z)
fake.persistent = True # Not to clear at backward
pred_fake = discriminator(fake)
loss_gen = F.mean(F.sigmoid_cross_entropy(
pred_fake, F.constant(1, pred_fake.shape)))
fake_dis = fake.unlinked()
pred_fake_dis = discriminator(fake_dis)
loss_dis = F.mean(F.sigmoid_cross_entropy(
pred_fake_dis, F.constant(0, pred_fake_dis.shape)))
# Real path
x = nn.Variable([args.batch_size, 1, 28, 28])
pred_real = discriminator(x)
loss_dis += F.mean(F.sigmoid_cross_entropy(pred_real,
F.constant(1, pred_real.shape)))
# Create Solver.
solver_gen = S.Adam(args.learning_rate, beta1=0.5)
solver_dis = S.Adam(args.learning_rate, beta1=0.5)
with nn.parameter_scope("gen"):
solver_gen.set_parameters(nn.get_parameters())
with nn.parameter_scope("dis"):
solver_dis.set_parameters(nn.get_parameters())
# Create monitor.
import nnabla.monitor as M
monitor = M.Monitor(args.monitor_path)
monitor_loss_gen = M.MonitorSeries("Generator loss", monitor, interval=10)
monitor_loss_dis = M.MonitorSeries(
"Discriminator loss", monitor, interval=10)
monitor_time = M.MonitorTimeElapsed("Time", monitor, interval=100)
monitor_fake = M.MonitorImageTile(
"Fake images", monitor, normalize_method=lambda x: x + 1 / 2.)
data = data_iterator_mnist(args.batch_size, True)
# Training loop.
for i in range(args.max_iter):
if i % args.model_save_interval == 0:
with nn.parameter_scope("gen"):
nn.save_parameters(os.path.join(
args.model_save_path, "generator_param_%06d.h5" % i))
with nn.parameter_scope("dis"):
nn.save_parameters(os.path.join(
args.model_save_path, "discriminator_param_%06d.h5" % i))
# Training forward
image, _ = data.next()
x.d = image / 255. - 0.5 # [0, 255] to [-1, 1]
z.d = np.random.randn(*z.shape)
# Generator update.
solver_gen.zero_grad()
loss_gen.forward(clear_no_need_grad=True)
loss_gen.backward(clear_buffer=True)
solver_gen.weight_decay(args.weight_decay)
solver_gen.update()
monitor_fake.add(i, fake)
monitor_loss_gen.add(i, loss_gen.d.copy())
# Discriminator update.
solver_dis.zero_grad()
loss_dis.forward(clear_no_need_grad=True)
loss_dis.backward(clear_buffer=True)
solver_dis.weight_decay(args.weight_decay)
solver_dis.update()
monitor_loss_dis.add(i, loss_dis.d.copy())
monitor_time.add(i)
nnp = os.path.join(
args.model_save_path, 'dcgan_%06d.nnp' % args.max_iter)
runtime_contents = {
'networks': [
{'name': 'Generator',
'batch_size': args.batch_size,
'outputs': {'G': fake},
'names': {'z': z}},
{'name': 'Discriminator',
'batch_size': args.batch_size,
'outputs': {'D': pred_real},
'names': {'x': x}}],
'executors': [
{'name': 'Generator',
'network': 'Generator',
'data': ['z'],
'output': ['G']},
{'name': 'Discriminator',
'network': 'Discriminator',
'data': ['x'],
'output': ['D']}]}
save.save(nnp, runtime_contents)
from cpp_forward_check import check_cpp_forward
check_cpp_forward(args.model_save_path, [z.d], [z], fake, nnp, "Generator")
def train():
"""
Main script.
Steps:
* Parse command line arguments.
* Specify a context for computation.
* Initialize DataIterator for MNIST.
* Construct a computation graph for training and validation.
* Initialize a solver and set parameter variables to it.
* Create monitor instances for saving and displaying training stats.
* Training loop
* Computate error rate for validation data (periodically)
* Get a next minibatch.
* Execute forwardprop on the training graph.
* Compute training error
* Set parameter gradients zero
* Execute backprop.
* Solver updates parameters by using gradients computed by backprop.
"""
args = get_args()
from numpy.random import seed
seed(0)
# Get context.
from nnabla.contrib.context import extension_context
extension_module = args.context
if args.context is None:
extension_module = 'cpu'
logger.info("Running in %s" % extension_module)
ctx = extension_context(extension_module, device_id=args.device_id)
nn.set_default_context(ctx)
# Create CNN network for both training and testing.
if args.net == 'lenet':
mnist_cnn_prediction = mnist_lenet_prediction
elif args.net == 'resnet':
mnist_cnn_prediction = mnist_resnet_prediction
else:
raise ValueError("Unknown network type {}".format(args.net))
# TRAIN
# Create input variables.
image = nn.Variable([args.batch_size, 1, 28, 28])
label = nn.Variable([args.batch_size, 1])
# Create prediction graph.
pred = mnist_cnn_prediction(image, test=False, aug=args.augment_train)
pred.persistent = True
# Create loss function.
loss = F.mean(F.softmax_cross_entropy(pred, label))
# TEST
# Create input variables.
vimage = nn.Variable([args.batch_size, 1, 28, 28])
vlabel = nn.Variable([args.batch_size, 1])
# Create predition graph.
vpred = mnist_cnn_prediction(vimage, test=True, aug=args.augment_test)
# Create Solver.
solver = S.Adam(args.learning_rate)
solver.set_parameters(nn.get_parameters())
# Create monitor.
from nnabla.monitor import Monitor, MonitorSeries, MonitorTimeElapsed
monitor = Monitor(args.monitor_path)
monitor_loss = MonitorSeries("Training loss", monitor, interval=10)
monitor_err = MonitorSeries("Training error", monitor, interval=10)
monitor_time = MonitorTimeElapsed("Training time", monitor, interval=100)
monitor_verr = MonitorSeries("Test error", monitor, interval=10)
# Initialize DataIterator for MNIST.
from numpy.random import RandomState
data = data_iterator_mnist(args.batch_size, True, rng=RandomState(1223))
vdata = data_iterator_mnist(args.batch_size, False)
# Training loop.
for i in range(args.max_iter):
if i % args.val_interval == 0:
# Validation
ve = 0.0
for j in range(args.val_iter):
vimage.d, vlabel.d = vdata.next()
vpred.forward(clear_buffer=True)
ve += categorical_error(vpred.d, vlabel.d)
monitor_verr.add(i, ve / args.val_iter)
if i % args.model_save_interval == 0:
nn.save_parameters(
os.path.join(args.model_save_path, 'params_%06d.h5' % i))
# Training forward
image.d, label.d = data.next()
solver.zero_grad()
loss.forward(clear_no_need_grad=True)
loss.backward(clear_buffer=True)
solver.weight_decay(args.weight_decay)
solver.update()
e = categorical_error(pred.d, label.d)
monitor_loss.add(i, loss.d.copy())
monitor_err.add(i, e)
monitor_time.add(i)
ve = 0.0
for j in range(args.val_iter):
vimage.d, vlabel.d = vdata.next()
vpred.forward(clear_buffer=True)
ve += categorical_error(vpred.d, vlabel.d)
monitor_verr.add(i, ve / args.val_iter)
parameter_file = os.path.join(
args.model_save_path,
'{}_params_{:06}.h5'.format(args.net, args.max_iter))
nn.save_parameters(parameter_file)
from nnabla.contrib.context import extension_context
from mnist_data import data_iterator_mnist
def mlp(image, test=False):
image /= 255.0
h = F.relu(PF.affine(image, 1000, name='l1'), inplace=True)
h = F.relu(PF.affine(h, 1000, name='l2'), inplace=True)
h = PF.affine(h, 10, name='l3')
return F.softmax(h)
# Get context.
ctx = extension_context('cpu', device_id=0)
nn.set_default_context(ctx)
# Create CNN network for both training and testing.
mnist_cnn_prediction = mlp
# Create input variables.
vimage = nn.Variable([1, 1, 28, 28])
vpred = mnist_cnn_prediction(vimage, test=True)
# Initialize DataIterator for MNIST.
vdata = data_iterator_mnist(1, False)
for j in tqdm.tqdm(range(vdata.size)):
vimage.d, _ = vdata.next()
vpred.forward(clear_buffer=True)
def infer():
"""
Main script.
Steps:
* Parse command line arguments.
* Specify a context for computation.
* Initialize DataIterator for MNIST.
* Construct a computation graph for inference.
* Load parameter variables to infer.
* Create monitor instances for saving and displaying infering stats.
"""
args = get_args()
from numpy.random import seed
seed(0)
# Get context.
from nnabla.ext_utils import get_extension_context
logger.info("Running in %s" % args.context)
ctx = get_extension_context(args.context,
device_id=args.device_id,
type_config=args.type_config)
nn.set_default_context(ctx)
# Create CNN network for both training and testing.
if args.net == 'lenet':
mnist_cnn_prediction = mnist_lenet_prediction
elif args.net == 'resnet':
mnist_cnn_prediction = mnist_resnet_prediction
else:
raise ValueError("Unknown network type {}".format(args.net))
# TEST
# Create input variables.
vimage = nn.Variable([args.batch_size, 1, 28, 28])
vlabel = nn.Variable([args.batch_size, 1])
# Create prediction graph.
vpred = mnist_cnn_prediction(vimage, test=True, aug=args.augment_test)
# Create Solver.
solver = S.Adam(args.learning_rate)
solver.set_parameters(nn.get_parameters())
# Create monitor.
from nnabla.monitor import Monitor, MonitorSeries, MonitorTimeElapsed
monitor = Monitor(args.monitor_path)
monitor_loss = MonitorSeries("Training loss", monitor, interval=10)
monitor_err = MonitorSeries("Training error", monitor, interval=10)
monitor_time = MonitorTimeElapsed("Training time", monitor, interval=100)
monitor_verr = MonitorSeries("Test error", monitor, interval=10)
# Initialize DataIterator for MNIST.
from numpy.random import RandomState
data = data_iterator_mnist(args.batch_size, True, rng=RandomState(1223))
vdata = data_iterator_mnist(1, False)
from nnabla.utils.nnp_graph import NnpLoader
# Read a .nnp file.
nnp = NnpLoader(args.pretrained)
# Assume a graph `graph_a` is in the nnp file.
net = nnp.get_network(nnp.get_network_names()[0], batch_size=1)
# `x` is an input of the graph.
x = net.inputs['x']
# 'y' is an outputs of the graph.
y = net.outputs['y']
ve = 0.0
for j in range(10000):
x.d, vlabel.d = vdata.next()
y.forward(clear_buffer=True)
ve += categorical_error(y.d, vlabel.d)
#monitor_verr.add(1, ve / args.val_iter)
print("acc=", 1 - ve / 10000, ".")
# append F.Softmax to the prediction graph so users see intuitive outputs
runtime_contents = {
'networks': [{
'name': 'Validation',
'batch_size': args.batch_size,
'outputs': {
'y': F.softmax(vpred)
},
'names': {
'x': vimage
}
}],
'executors': [{
'name': 'Runtime',
'network': 'Validation',
'data': ['x'],
'output': ['y']
}]
}
def train():
"""
Main script.
Steps:
* Parse command line arguments.
* Specify a context for computation.
* Initialize DataIterator for MNIST.
* Construct a computation graph for training and validation.
* Initialize a solver and set parameter variables to it.
* Create monitor instances for saving and displaying training stats.
* Training loop
* Computate error rate for validation data (periodically)
* Get a next minibatch.
* Set parameter gradients zero
* Execute forwardprop on the training graph.
* Execute backprop.
* Solver updates parameters by using gradients computed by backprop.
* Compute training error
"""
args = get_args(monitor_path='tmp.monitor.bnn')
# Get context.
from nnabla.ext_utils import get_extension_context
logger.info("Running in %s" % args.context)
ctx = get_extension_context(
args.context, device_id=args.device_id, type_config=args.type_config)
nn.set_default_context(ctx)
# Initialize DataIterator for MNIST.
data = data_iterator_mnist(args.batch_size, True)
vdata = data_iterator_mnist(args.batch_size, False)
# Create CNN network for both training and testing.
mnist_cnn_prediction = mnist_binary_connect_lenet_prediction
if args.net == 'bincon':
mnist_cnn_prediction = mnist_binary_connect_lenet_prediction
elif args.net == 'binnet':
mnist_cnn_prediction = mnist_binary_net_lenet_prediction
elif args.net == 'bwn':
mnist_cnn_prediction = mnist_binary_weight_lenet_prediction
elif args.net == 'bincon_resnet':
mnist_cnn_prediction = mnist_binary_connect_resnet_prediction
elif args.net == 'binnet_resnet':
mnist_cnn_prediction = mnist_binary_net_resnet_prediction
elif args.net == 'bwn_resnet':
mnist_cnn_prediction = mnist_binary_weight_resnet_prediction
# TRAIN
# Create input variables.
image = nn.Variable([args.batch_size, 1, 28, 28])
label = nn.Variable([args.batch_size, 1])
# Create predition graph.
pred = mnist_cnn_prediction(image / 255, test=False)
pred.persistent = True
# Create loss function.
loss = F.mean(F.softmax_cross_entropy(pred, label))
# TEST
# Create input variables.
vimage = nn.Variable([args.batch_size, 1, 28, 28])
vlabel = nn.Variable([args.batch_size, 1])
# Create predition graph.
vpred = mnist_cnn_prediction(vimage / 255, test=True)
# Create Solver.
solver = S.Adam(args.learning_rate)
solver.set_parameters(nn.get_parameters())
# Create monitor.
import nnabla.monitor as M
monitor = M.Monitor(args.monitor_path)
monitor_loss = M.MonitorSeries("Training loss", monitor, interval=10)
monitor_err = M.MonitorSeries("Training error", monitor, interval=10)
monitor_time = M.MonitorTimeElapsed("Training time", monitor, interval=100)
monitor_verr = M.MonitorSeries("Test error", monitor, interval=10)
# Training loop.
for i in range(args.max_iter):
if i % args.val_interval == 0:
# Validation
ve = 0.0
for j in range(args.val_iter):
vimage.d, vlabel.d = vdata.next()
vpred.forward(clear_buffer=True)
ve += categorical_error(vpred.d, vlabel.d)
monitor_verr.add(i, ve / args.val_iter)
if i % args.model_save_interval == 0:
nn.save_parameters(os.path.join(
args.model_save_path, 'params_%06d.h5' % i))
# Training forward
image.d, label.d = data.next()
solver.zero_grad()
loss.forward(clear_no_need_grad=True)
# Training backward & update
loss.backward(clear_buffer=True)
solver.weight_decay(args.weight_decay)
solver.update()
# Monitor
e = categorical_error(pred.d, label.d)
monitor_loss.add(i, loss.d.copy())
monitor_err.add(i, e)
monitor_time.add(i)
parameter_file = os.path.join(
args.model_save_path, 'params_%06d.h5' % args.max_iter)
nn.save_parameters(parameter_file)
def main():
"""
Main script.
Steps:
* Get and set context.
* Load Dataset
* Initialize DataIterator.
* Create Networks
* Net for Labeled Data
* Net for Unlabeled Data
* Net for Test Data
* Create Solver.
* Training Loop.
* Test
* Training
* by Labeled Data
* Calculate Supervised Loss
* by Unlabeled Data
* Calculate Virtual Adversarial Noise
* Calculate Unsupervised Loss
"""
args = get_args()
# Get context.
from nnabla.ext_utils import get_extension_context
logger.info("Running in %s" % args.context)
ctx = get_extension_context(args.context,
device_id=args.device_id,
type_config=args.type_config)
nn.set_default_context(ctx)
shape_x = (1, 28, 28)
n_h = args.n_units
n_y = args.n_class
# Load MNIST Dataset
from mnist_data import load_mnist, data_iterator_mnist
images, labels = load_mnist(train=True)
rng = np.random.RandomState(706)
inds = rng.permutation(len(images))
def feed_labeled(i):
j = inds[i]
return images[j], labels[j]
def feed_unlabeled(i):
j = inds[i]
return images[j], labels[j]
di_l = data_iterator_simple(feed_labeled,
args.n_labeled,
args.batchsize_l,
shuffle=True,
rng=rng,
with_file_cache=False)
di_u = data_iterator_simple(feed_unlabeled,
args.n_train,
args.batchsize_u,
shuffle=True,
rng=rng,
with_file_cache=False)
di_v = data_iterator_mnist(args.batchsize_v, train=False)
# Create networks
# feed-forward-net building function
def forward(x, test=False):
return mlp_net(x, n_h, n_y, test)
# Net for learning labeled data
xl = nn.Variable((args.batchsize_l, ) + shape_x, need_grad=False)
yl = forward(xl, test=False)
tl = nn.Variable((args.batchsize_l, 1), need_grad=False)
loss_l = F.mean(F.softmax_cross_entropy(yl, tl))
# Net for learning unlabeled data
xu = nn.Variable((args.batchsize_u, ) + shape_x, need_grad=False)
yu = forward(xu, test=False)
y1 = yu.get_unlinked_variable()
y1.need_grad = False
noise = nn.Variable((args.batchsize_u, ) + shape_x, need_grad=True)
r = noise / (F.sum(noise**2, [1, 2, 3], keepdims=True))**0.5
r.persistent = True
y2 = forward(xu + args.xi_for_vat * r, test=False)
y3 = forward(xu + args.eps_for_vat * r, test=False)
loss_k = F.mean(distance(y1, y2))
loss_u = F.mean(distance(y1, y3))
# Net for evaluating validation data
xv = nn.Variable((args.batchsize_v, ) + shape_x, need_grad=False)
hv = forward(xv, test=True)
tv = nn.Variable((args.batchsize_v, 1), need_grad=False)
err = F.mean(F.top_n_error(hv, tv, n=1))
# Create solver
solver = S.Adam(args.learning_rate)
solver.set_parameters(nn.get_parameters())
# Monitor training and validation stats.
import nnabla.monitor as M
monitor = M.Monitor(args.model_save_path)
monitor_verr = M.MonitorSeries("Test error", monitor, interval=240)
monitor_time = M.MonitorTimeElapsed("Elapsed time", monitor, interval=240)
# Training Loop.
t0 = time.time()
for i in range(args.max_iter):
# Validation Test
if i % args.val_interval == 0:
valid_error = calc_validation_error(di_v, xv, tv, err,
args.val_iter)
monitor_verr.add(i, valid_error)
#################################
## Training by Labeled Data #####
#################################
# forward, backward and update
xl.d, tl.d = di_l.next()
xl.d = xl.d / 255
solver.zero_grad()
loss_l.forward(clear_no_need_grad=True)
loss_l.backward(clear_buffer=True)
solver.weight_decay(args.weight_decay)
solver.update()
#################################
## Training by Unlabeled Data ###
#################################
# Calculate y without noise, only once.
xu.d, _ = di_u.next()
xu.d = xu.d / 255
yu.forward(clear_buffer=True)
##### Calculate Adversarial Noise #####
# Do power method iteration
noise.d = np.random.normal(size=xu.shape).astype(np.float32)
for k in range(args.n_iter_for_power_method):
r.grad.zero()
loss_k.forward(clear_no_need_grad=True)
loss_k.backward(clear_buffer=True)
noise.data.copy_from(r.grad)
##### Calculate loss for unlabeled data #####
# forward, backward and update
solver.zero_grad()
loss_u.forward(clear_no_need_grad=True)
loss_u.backward(clear_buffer=True)
solver.weight_decay(args.weight_decay)
solver.update()
##### Learning rate update #####
if i % args.iter_per_epoch == 0:
solver.set_learning_rate(solver.learning_rate() *
args.learning_rate_decay)
monitor_time.add(i)
# Evaluate the final model by the error rate with validation dataset
valid_error = calc_validation_error(di_v, xv, tv, err, args.val_iter)
monitor_verr.add(i, valid_error)
monitor_time.add(i)
# Save the model.
parameter_file = os.path.join(args.model_save_path,
'params_%06d.h5' % args.max_iter)
nn.save_parameters(parameter_file)
def classification_svd():
args = get_args()
# Get context.
from nnabla.ext_utils import get_extension_context
logger.info("Running in %s" % args.context)
ctx = get_extension_context(args.context,
device_id=args.device_id,
type_config=args.type_config)
nn.set_default_context(ctx)
# Create CNN network for both training and testing.
mnist_cnn_prediction = mnist_lenet_prediction_slim
# TRAIN
reference = "reference"
slim = "slim"
rrate = 0.5 # reduction rate
# Create input variables.
image = nn.Variable([args.batch_size, 1, 28, 28])
label = nn.Variable([args.batch_size, 1])
# Create `reference` and "slim" prediction graph.
model_load_path = args.model_load_path
pred = mnist_cnn_prediction(image, scope=slim, rrate=rrate, test=False)
pred.persistent = True
# Decompose and set parameters
decompose_network_and_set_params(model_load_path, reference, slim, rrate)
loss = F.mean(F.softmax_cross_entropy(pred, label))
# TEST
# Create input variables.
vimage = nn.Variable([args.batch_size, 1, 28, 28])
vlabel = nn.Variable([args.batch_size, 1])
# Create reference prediction graph.
vpred = mnist_cnn_prediction(vimage, scope=slim, rrate=rrate, test=True)
# Create Solver.
solver = S.Adam(args.learning_rate)
with nn.parameter_scope(slim):
solver.set_parameters(nn.get_parameters())
# Create monitor.
from nnabla.monitor import Monitor, MonitorSeries, MonitorTimeElapsed
monitor = Monitor(args.monitor_path)
monitor_loss = MonitorSeries("Training loss", monitor, interval=10)
monitor_err = MonitorSeries("Training error", monitor, interval=10)
monitor_time = MonitorTimeElapsed("Training time", monitor, interval=100)
monitor_verr = MonitorSeries("Test error", monitor, interval=10)
# Initialize DataIterator for MNIST.
data = data_iterator_mnist(args.batch_size, True)
vdata = data_iterator_mnist(args.batch_size, False)
best_ve = 1.0
# Training loop.
for i in range(args.max_iter):
if i % args.val_interval == 0:
# Validation
ve = 0.0
for j in range(args.val_iter):
vimage.d, vlabel.d = vdata.next()
vpred.forward(clear_buffer=True)
ve += categorical_error(vpred.d, vlabel.d)
monitor_verr.add(i, ve / args.val_iter)
if ve < best_ve:
nn.save_parameters(
os.path.join(args.model_save_path, 'params_%06d.h5' % i))
best_ve = ve
# Training forward
image.d, label.d = data.next()
solver.zero_grad()
loss.forward(clear_no_need_grad=True)
loss.backward(clear_buffer=True)
solver.weight_decay(args.weight_decay)
solver.update()
e = categorical_error(pred.d, label.d)
monitor_loss.add(i, loss.d.copy())
monitor_err.add(i, e)
monitor_time.add(i)
ve = 0.0
for j in range(args.val_iter):
vimage.d, vlabel.d = vdata.next()
vpred.forward(clear_buffer=True)
ve += categorical_error(vpred.d, vlabel.d)
monitor_verr.add(i, ve / args.val_iter)
parameter_file = os.path.join(args.model_save_path,
'params_{:06}.h5'.format(args.max_iter))
nn.save_parameters(parameter_file)
def train(args):
"""
Main script.
"""
# Get context.
from nnabla.contrib.context import extension_context
extension_module = args.context
if args.context is None:
extension_module = 'cpu'
logger.info("Running in %s" % extension_module)
ctx = extension_context(extension_module, device_id=args.device_id)
nn.set_default_context(ctx)
# Create CNN network for both training and testing.
# TRAIN
# Fake path
z = nn.Variable([args.batch_size, 100, 1, 1])
fake = generator(z)
fake.persistent = True # Not to clear at backward
pred_fake = discriminator(fake)
loss_gen = F.mean(
F.sigmoid_cross_entropy(pred_fake, F.constant(1, pred_fake.shape)))
fake_dis = fake.unlinked()
pred_fake_dis = discriminator(fake_dis)
loss_dis = F.mean(
F.sigmoid_cross_entropy(pred_fake_dis,
F.constant(0, pred_fake_dis.shape)))
# Real path
x = nn.Variable([args.batch_size, 1, 28, 28])
pred_real = discriminator(x)
loss_dis += F.mean(
F.sigmoid_cross_entropy(pred_real, F.constant(1, pred_real.shape)))
# Create Solver.
solver_gen = S.Adam(args.learning_rate, beta1=0.5)
solver_dis = S.Adam(args.learning_rate, beta1=0.5)
with nn.parameter_scope("gen"):
solver_gen.set_parameters(nn.get_parameters())
with nn.parameter_scope("dis"):
solver_dis.set_parameters(nn.get_parameters())
# Create monitor.
import nnabla.monitor as M
monitor = M.Monitor(args.monitor_path)
monitor_loss_gen = M.MonitorSeries("Generator loss", monitor, interval=10)
monitor_loss_dis = M.MonitorSeries("Discriminator loss",
monitor,
interval=10)
monitor_time = M.MonitorTimeElapsed("Time", monitor, interval=100)
monitor_fake = M.MonitorImageTile("Fake images",
monitor,
normalize_method=lambda x: x + 1 / 2.)
data = data_iterator_mnist(args.batch_size, True)
# Training loop.
for i in range(args.max_iter):
if i % args.model_save_interval == 0:
with nn.parameter_scope("gen"):
nn.save_parameters(
os.path.join(args.model_save_path,
"generator_param_%06d.h5" % i))
with nn.parameter_scope("dis"):
nn.save_parameters(
os.path.join(args.model_save_path,
"discriminator_param_%06d.h5" % i))
# Training forward
image, _ = data.next()
x.d = image / 255. - 0.5 # [0, 255] to [-1, 1]
z.d = np.random.randn(*z.shape)
# Generator update.
solver_gen.zero_grad()
loss_gen.forward(clear_no_need_grad=True)
loss_gen.backward(clear_buffer=True)
solver_gen.weight_decay(args.weight_decay)
solver_gen.update()
monitor_fake.add(i, fake)
monitor_loss_gen.add(i, loss_gen.d.copy())
# Discriminator update.
solver_dis.zero_grad()
loss_dis.forward(clear_no_need_grad=True)
loss_dis.backward(clear_buffer=True)
solver_dis.weight_decay(args.weight_decay)
solver_dis.update()
monitor_loss_dis.add(i, loss_dis.d.copy())
monitor_time.add(i)
nnp = os.path.join(args.model_save_path, 'dcgan_%06d.nnp' % args.max_iter)
runtime_contents = {
'networks': [{
'name': 'Generator',
'batch_size': args.batch_size,
'outputs': {
'G': fake
},
'names': {
'z': z
}
}, {
'name': 'Discriminator',
'batch_size': args.batch_size,
'outputs': {
'D': pred_real
},
'names': {
'x': x
}
}],
'executors': [{
'name': 'Generator',
'network': 'Generator',
'data': ['z'],
'output': ['G']
}, {
'name': 'Discriminator',
'network': 'Discriminator',
'data': ['x'],
'output': ['D']
}]
}
save.save(nnp, runtime_contents)
from cpp_forward_check import check_cpp_forward
check_cpp_forward(args.model_save_path, [z.d], [z], fake, nnp, "Generator")
