def train(net,
train_iter,
test_iter,
loss,
trainer,
num_epochs,
ctx_list=d2l.try_all_gpus()):
num_batches, timer = len(train_iter), d2l.Timer()
for epoch in range(num_epochs):
# store training_loss, training_accuracy, num_examples, num_features
metric = [0.0] * 4
for i, (features, labels) in enumerate(train_iter):
timer.start()
l, acc = d2l.train_batch_ch12(net, features, labels, loss, trainer,
ctx_list)
metric = [
a + b
for a, b in zip(metric, (l, acc, labels.shape[0], labels.size))
]
timer.stop()
if (i + 1) % (num_batches // 5) == 0:
print(epoch + i / num_batches,
(metric[0] / metric[2], metric[1] / metric[3], None))
test_acc = d2l.evaluate_accuracy_gpus(net, test_iter)
print('loss %.3f, train acc %.3f, test acc %.3f' %
(metric[0] / metric[2], metric[1] / metric[3], test_acc))
print('%.1f exampes/sec on %s' %
(metric[2] * num_epochs / timer.sum(), ctx_list))
def train_recsys_rating(net, train_iter, test_iter, loss, trainer, num_epochs,
ctx_list=d2l.try_all_gpus(), evaluator=None,
**kwargs):
timer = d2l.Timer()
animator = d2l.Animator(xlabel='epoch', xlim=[1, num_epochs], ylim=[0, 2],
legend=['train loss', 'test RMSE'])
for epoch in range(num_epochs):
metric, l = d2l.Accumulator(3), 0.
for i, values in enumerate(train_iter):
timer.start()
input_data = []
values = values if isinstance(values, list) else [values]
for v in values:
input_data.append(gluon.utils.split_and_load(v, ctx_list))
train_feat = input_data[0:-1] if len(values) > 1 else input_data
train_label = input_data[-1]
with autograd.record():
preds = [net(*t) for t in zip(*train_feat)]
ls = [loss(p, s) for p, s in zip(preds, train_label)]
[l.backward() for l in ls]
l += sum([l.asnumpy() for l in ls]).mean() / len(ctx_list)
trainer.step(values[0].shape[0])
metric.add(l, values[0].shape[0], values[0].size)
timer.stop()
if len(kwargs) > 0: # it will be used in section AutoRec.
test_rmse = evaluator(net, test_iter, kwargs['inter_mat'],
ctx_list)
else:
test_rmse = evaluator(net, test_iter, ctx_list)
train_l = l / (i + 1)
animator.add(epoch + 1, (train_l, test_rmse))
print('train loss %.3f, test RMSE %.3f'
% (metric[0] / metric[1], test_rmse))
print('%.1f examples/sec on %s'
% (metric[2] * num_epochs / timer.sum(), ctx_list))
def train_s2s_ch9(model, data_iter, lr, num_epochs, ctx):
model.initialize(init.Xavier(), force_reinit=True, ctx=ctx)
trainer = gluon.Trainer(model.collect_params(),'adam', {'learning_rate': lr})
loss = MaskedSoftmaxCELoss()
animator = d2l.Animator(xlabel='epoch', ylabel='loss',xlim=[1, num_epochs], ylim=[0, 0.25])
animator.fig.subplots_adjust(hspace=0.3) #FIXME - TEST
for epoch in range(1, num_epochs + 1):
timer = d2l.Timer()
metric = d2l.Accumulator(2) # loss_sum, num_tokens
for batch in data_iter:
X, X_vlen, Y, Y_vlen = [x.as_in_context(ctx) for x in batch]
Y_input, Y_label, Y_vlen = Y[:, :-1], Y[:, 1:], Y_vlen-1
with autograd.record():
Y_hat, _ = model(X, Y_input, X_vlen, Y_vlen)
l = loss(Y_hat, Y_label, Y_vlen)
l.backward()
d2l.grad_clipping(model, 1)
num_tokens = Y_vlen.sum()
trainer.step(num_tokens)
metric.add(l.sum(), num_tokens)
if epoch % 10 == 0:
animator.add(epoch, (metric[0]/metric[1],))
print('loss %.3f, %d tokens/sec on %s ' % (
metric[0]/metric[1], metric[1]/timer.stop(), ctx))
def train_ch13(net,
train_iter,
test_iter,
loss,
trainer,
num_epochs,
ctx_list=d2l.try_all_gpus(),
split_f=d2l.split_batch):
num_batches, timer = len(train_iter), d2l.Timer()
animator = d2l.Animator(xlabel='epoch',
xlim=[0, num_epochs],
ylim=[0, 1],
legend=['train loss', 'train acc', 'test acc'])
for epoch in range(num_epochs):
# Store training_loss, training_accuracy, num_examples, num_features
metric = d2l.Accumulator(4)
for i, (features, labels) in enumerate(train_iter):
timer.start()
l, acc = train_batch_ch13(net, features, labels, loss, trainer,
ctx_list, split_f)
metric.add(l, acc, labels.shape[0], labels.size)
timer.stop()
if (i + 1) % (num_batches // 5) == 0:
animator.add(
epoch + i / num_batches,
(metric[0] / metric[2], metric[1] / metric[3], None))
test_acc = d2l.evaluate_accuracy_gpus(net, test_iter, split_f)
animator.add(epoch + 1, (None, None, test_acc))
print('loss %.3f, train acc %.3f, test acc %.3f' %
(metric[0] / metric[2], metric[1] / metric[3], test_acc))
print('%.1f examples/sec on %s' %
(metric[2] * num_epochs / timer.sum(), ctx_list))
def speed():
timer = d2l.Timer()
c = np.zeros(n)
for i in range(n):
c[i] = a[i] + b[i]
t1 = float(timer.stop())
timer.start()
d = a + b
t2 = float(timer.stop())
print("d size {}".format(d.size))
print('for loop : %.5f sec' % t1)
print('vectors : %.10f sec' % t2)
if t2 > 0:
print('ratio : %.1f' % float(t1 / t2))
def train_ranking(net,
train_iter,
test_iter,
loss,
trainer,
test_seq_iter,
num_users,
num_items,
num_epochs,
ctx_list,
evaluator,
candidates,
eval_step=1):
timer, hit_rate, auc = d2l.Timer(), 0, 0
animator = d2l.Animator(xlabel='epoch',
xlim=[1, num_epochs],
ylim=[0, 1],
legend=['test hit rate', 'test AUC'])
for epoch in range(num_epochs):
metric, l = d2l.Accumulator(3), 0.
for i, values in enumerate(train_iter):
input_data = []
for v in values:
input_data.append(gluon.utils.split_and_load(v, ctx_list))
with autograd.record():
p_pos = [net(*t) for t in zip(*input_data[0:-1])]
p_neg = [
net(*t) for t in zip(*input_data[0:-2], input_data[-1])
]
ls = [loss(p, n) for p, n in zip(p_pos, p_neg)]
[l.backward(retain_graph=False) for l in ls]
l += sum([l.asnumpy() for l in ls]).mean() / len(ctx_list)
trainer.step(values[0].shape[0])
metric.add(l, values[0].shape[0], values[0].size)
timer.stop()
with autograd.predict_mode():
if (epoch + 1) % eval_step == 0:
hit_rate, auc = evaluator(net, test_iter, test_seq_iter,
candidates, num_users, num_items,
ctx_list)
animator.add(epoch + 1, (hit_rate, auc))
print('train loss %.3f, test hit rate %.3f, test AUC %.3f' %
(metric[0] / metric[1], hit_rate, auc))
print('%.1f examples/sec on %s' %
(metric[2] * num_epochs / timer.sum(), ctx_list))
net.initialize()
for layer in net:
X = layer(X)
print(layer.name, 'output shape:\t', X.shape)
## =============== TRAINING THE NET ====================
num_epochs, lr, wd, ctx = 10, 0.1, 1e-3, d2l.try_all_gpus()
loss = gluon.loss.SoftmaxCrossEntropyLoss()
net.collect_params().reset_ctx(ctx)
trainer = gluon.Trainer(net.collect_params(), 'sgd', {
'learning_rate': lr,
'wd': wd
})
num_batches, timer = len(DataLoader_Single_train), d2l.Timer()
animator = d2l.Animator(
xlabel='epoch',
xlim=[0, num_epochs],
ylim=[0, 1],
legend=['train loss', 'train acc', 'val loss', 'val acc'])
ctx = d2l.try_all_gpus()
for epoch in range(num_epochs):
# Training loop
# Store training_loss, training_accuracy, num_examples, num_features
metric = d2l.Accumulator(4)
for i, (Xs_in, ys_in) in enumerate(DataLoader_Single_train):
print("Training iteration: " + str(i))
timer.start()
Xs = gluon.utils.split_and_load(Xs_in.astype("float32"), ctx)
npx.set_np()
d2l.use_svg_display()
mnist_train = gluon.data.vision.FashionMNIST(train=True)
mnist_test = gluon.data.vision.FashionMNIST(train=False)
print("train length : {}, test length: {}".format(len(mnist_train),
len(mnist_test)))
X, y = mnist_train[:18]
# d2l.show_images(X.squeeze(axis=-1), 2, 9, titles=d2l.get_fashion_mnist_labels(y))
# plt.show()
batch_size = 256
transformer = gluon.data.vision.transforms.ToTensor()
train_iter = gluon.data.DataLoader(mnist_train.transform_first(transformer),
batch_size,
shuffle=True,
num_workers=d2l.get_dataloader_workers())
timer = d2l.Timer()
for X, y in train_iter:
continue
print("loading dada takes {:.2f} sec".format(timer.stop()))
train_iter, test_iter = d2l.load_data_fashion_mnist(32, (64, 64))
for X, y in train_iter:
print(X.shape)
break