def train_cpu(net,
train_iter,
test_iter,
lr,
num_epochs,
batch_size,
params,
trainer=None):
for epoch in range(1, num_epochs + 1):
train_sum_loss = 0
train_sum_acc = 0
for X, y in train_iter:
with autograd.record():
y_hat = net(X)
l = loss(y_hat, y)
l.backward()
if trainer is None:
sgd(params, lr, batch_size)
train_sum_loss += l.mean().asscalar()
train_sum_acc += accuarcy(y_hat, y)
print("epoch: %d, loss: %f, train_acc: %f" %
(epoch, train_sum_loss / len(train_iter),
train_sum_acc / len(train_iter)))
test_acc = test_accuarcy(net, test_iter)
print("test_acc: %f" % test_acc)
def validate(opts, val_list, file, val_iter, sw=None, num_plot=None):
"""Validation"""
if num_plot is None:
num_plot = val_list.__len__()
loss_cumulative = []
pred_list = []
lab_list = []
val_iter.reset()
count = 0 # count number of pred and lab retained
for (i, batch) in enumerate(val_iter):
idx = batch.data[0].asnumpy()
data = nd.zeros((args.batch_size, 60, 20, 240, 240), ctx=ctx)
label = nd.zeros((args.batch_size, 20, 240, 240), ctx=ctx)
for (j, jj) in zip(idx, range(opts.batch_size)):
data[jj] = np.load("%sIMG.npy" % val_list[int(j)])
label[jj] = np.load("%s%s.npy" % (val_list[int(j)], file))
pred = net(data)
L = loss(pred, label)
loss_cumulative.append((L.expand_dims(1)))
if count < num_plot:
pred_list.append(pred)
lab_list.append(label)
count += 1
return nd.concat(*loss_cumulative, dim=0).mean().asscalar(), \
nd.squeeze(nd.concat(*pred_list, dim=0)), \
nd.squeeze(nd.concat(*lab_list, dim=0))
ones_like = F.ones_like
if not self._from_logits:
pt = softmax(pt, axis=self._axis)
if not isinstance(_alpha, mx.base.numeric_types):
_alpha = F.broadcast_mul(ones_like(pt),_alpha)
_alpha = pick(_alpha, label, axis=self._axis, keepdims=True)
if self._sparse_label:
pt = pick(pt, label, axis=self._axis, keepdims=True)
else:
label = _reshape_like(F, label, pt)
pt = (pt * label).sum(axis=self._axis, keepdims=True)
loss = _apply_gamma_alpha(F, pt, self._gamma, _alpha)
if is_np_array():
if F is mx.ndarray:
return loss.mean(axis=tuple(range(1, loss.ndim)))
else:
return F.npx.batch_flatten(loss).mean(axis=1)
else:
return loss.mean(axis=self._batch_axis, exclude=True)
if __name__ == '__main__':
x = mx.nd.array([[1,2,3],[4,5,6],[9,8,7]])
label = mx.nd.array([[1],[2],[0]])
alpha = [1,2,3]
loss = FocalLoss(alpha=alpha)
loss.hybridize()
print(loss(x,label))
# 默认初始化的是权重参数,采用标准差为0.01的正态分布
# 偏差参数默认初始化为0
# 这时net已经有了w和b，后续操作中只需要将输入输入层即可
# 这个net(X)的操作就是求出预测值的过程！
net.initialize(init.Normal(sigma=0.01))
# 【定义损失函数】
loss = loss.L2Loss() # 平方损失又称为L2范数损失
# 【定义优化算法】
# 定义一个模型参数优化算法一共需要4个参数，w,b,learning_rate,batch_size
# 其中batch_size这个参数在最后的step函数中给出！
trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.03})
# 【训练模型】
num_epochs = 3
for epoch in range(num_epochs):
for X, y in data_iter:
with autograd.record():
l = loss(net(X), y)
l.backward()
# 打印net第一层中的权重和偏差信息！
print(net[0].weight.data(), net[0].bias.data())
trainer.step(batch_size)
l = loss(net(features), labels)
print('epoch %d, loss %f' % (epoch, l.mean().asnumpy()))
def train(opts, net, sw):
"""Training"""
first_val = False
train_list, val_list = get_subject_list(opts)
trainer = set_trainer(opts, net)
file, dir_checkpoints = prepare_dir(opts)
train_iter, val_iter = set_iter(opts)
tic = time.time()
lowest_loss = np.Inf
for epoch in range(opts.epochs):
etic = time.time()
btic = time.time()
train_iter.reset()
loss_cumulative = []
for (i, batch) in enumerate(train_iter):
idx = batch.data[0].asnumpy()
data = nd.zeros((args.batch_size, 60, 20, 240, 240), ctx=ctx)
label = nd.zeros((args.batch_size, 20, 240, 240), ctx=ctx)
for (j, jj) in zip(idx, range(opts.batch_size)):
data[jj] = np.load("%sIMG.npy" % train_list[int(j)])
label[jj] = np.load("%s%s.npy" % (train_list[int(j)], file))
with autograd.record():
pred = net(data)
L = loss(pred, label)
L.backward()
loss_cumulative.append((L.expand_dims(1)))
trainer.step(opts.batch_size)
if (i + 1) % opts.log_interval == 0:
print(
'[Epoch {}/{}] [Batch {}] Loss: {:.3f}, {:.2f} Samples / sec'
.format(
epoch, opts.epochs, i, L.asscalar(),
opts.log_interval * opts.batch_size /
(time.time() - btic)))
btic = time.time()
loss_train = nd.concat(*loss_cumulative, dim=0).mean().asscalar()
sw.add_scalar('loss', ('training', loss_train), global_step=epoch)
if (epoch + 1) % opts.val_interval == 0:
loss_val, preds, labs = validate(opts, val_list, file, val_iter,
sw)
sw.add_scalar('loss', ('validation', loss_val), global_step=epoch)
# Log validation predictions
for k in range(preds.shape[0]):
im = integrate_pred_lab(integrate_slices(norm01(preds[k])),
integrate_slices(norm01(labs[k])))
sw.add_image('val_predictions_%02d' % k, im, global_step=epoch)
if loss_val <= lowest_loss:
lowest_loss = loss_val
np.save("%sval_pred_best_epoch" % opts.dir_out,
preds.asnumpy())
np.save("%sval_lab_best_epoch" % opts.dir_out, labs.asnumpy())
best_epoch = epoch
if loss_val == lowest_loss:
net.save_params("%sBest_epoch.params" % dir_checkpoints)
first_val = True
# Log training predictions
im = integrate_pred_lab(
integrate_slices(norm01(nd.squeeze(pred[-1]))),
integrate_slices(norm01(nd.squeeze(label[-1]))))
sw.add_image('train_predictions', norm01(im), global_step=epoch)
sw_cmd = "cd '%s' \ntensorboard --logdir=./logs --host=127.0.0.1 --port=8888" % opts.dir_out
print('Copy paste this to view Tensorboard:\n%s' % sw_cmd)
print('Training loss: {:.2f}'.format(loss_train))
if first_val and ((epoch + 1) % opts.checkpoint_interval == 0):
net.save_params("%sEpoch%03d.params" % (dir_checkpoints, epoch))
print('Validation loss: {:.2f}, '
'\nBest Validation Loss: {:.2f} at epoch {:03d}'
'\nEpoch time: {:.2f} secs'.format(loss_val, lowest_loss,
best_epoch,
time.time() - etic))
print('Total training time: {:.2f} secs'.format(time.time() - tic))
def train(opts, model, sw):
"""Training"""
train_list, val_list = get_subject_list(opts)
trainer = set_trainer(opts, model)
file, dir_checkpoints = prepare_dir(opts)
train_iter, val_iter = set_iter(opts)
tic = time.time()
lowest_loss = np.Inf
global_step = 0
for epoch in range(opts.epochs):
sw_cmd = "cd '%s' \ntensorboard --logdir=./logs --host=127.0.0.1 --port=8888" % opts.dir_out
print('Copy paste this to view Tensorboard:\n%s' % sw_cmd)
etic = time.time()
btic = time.time()
train_iter.reset()
loss_cumulative = []
for (i, batch) in enumerate(train_iter):
idx = batch.data[0].asnumpy()
data2d = np.load("%sIMG.npy" % train_list[int(idx)])
label2d = np.load("%s%s.npy" % (train_list[int(idx)], file))
data = nd.array(_reshape(data2d[..., 80:112, 80:112],
batch_size=opts.batch_size1),
ctx=ctx) # [..., 80:112, 80:112]
label = nd.array(_reshape(label2d[..., 80:112, 80:112],
batch_size=opts.batch_size1),
ctx=ctx)
preds = nd.zeros(label.shape)
# hidden = detach(hidden)
for i_batch in range(data.shape[0]):
hidden = model.begin_state(func=nd.zeros,
batch_size=opts.batch_size1,
ctx=ctx)
with autograd.record():
# pred = net(data)
pred, hidden = model(data[i_batch], hidden)
L = loss(pred, label[i_batch])
R = np.corrcoef(pred.asnumpy().flatten(),
label[i_batch].asnumpy().flatten())[0, 1]
print(R)
L.backward()
# grads = [i.grad(ctx) for i in model.collect_params().values()]
# utils.clip_global_norm(grads, opts.clip * opts.batch_size1)
trainer.step(opts.batch_size1)
loss_cumulative.append((L.expand_dims(1)))
preds[i_batch] = pred
if (i_batch + 1) == preds.shape[0]:
# if (i_batch + 1) == 1:
u0 = _shape_recover(label, (1, 20, 32, 32))
u1 = _shape_recover(preds,
(1, 20, 32, 32)).as_in_context(ctx)
v0 = u0[0, 0]
v1 = u1[0, 0]
for kk in range(1, u0.shape[1]):
v0 = nd.concat(v0, u0[0, kk], dim=-1)
v1 = nd.concat(v1, u1[0, kk], dim=-1)
vv = nd.concat(v0, v1, dim=-2)
sw.add_image('label_prediction',
norm01(vv),
global_step=global_step)
global_step += 1
plt.show()
if (i + 1) % opts.log_interval == 0:
print(
'[Epoch {}/{}] [Batch {}-{}] Loss: {:.3f}, {:.2f} Samples / sec'
.format(
epoch, opts.epochs, i, i_batch,
L.mean().asscalar(), opts.log_interval *
opts.batch_size1 / (time.time() - btic)))
btic = time.time()
loss_train = nd.concat(*loss_cumulative, dim=0).mean().asscalar()
sw.add_scalar('loss', ('training', loss_train), global_step=epoch)
loss_val = lowest_loss
if opts.val_interval & (epoch + 1) % opts.val_interval == 0:
loss_val, preds, labs = validate(opts, val_list, file, val_iter,
sw)
sw.add_scalar('loss', ('validation', loss_val), global_step=epoch)
# Log validation predictions
for k in range(preds.shape[0]):
im = integrate_pred_lab(integrate_slices(norm01(preds[k])),
integrate_slices(norm01(labs[k])))
sw.add_image('val_predictions_%02d' % k, im, global_step=epoch)
if loss_val <= lowest_loss:
lowest_loss = loss_val
np.save("%sval_pred_best_epoch" % opts.dir_out,
preds.asnumpy())
np.save("%sval_lab_best_epoch" % opts.dir_out, labs.asnumpy())
best_epoch = epoch
# Log training predictions
im = integrate_pred_lab(
integrate_slices(norm01(nd.squeeze(pred[-1]))),
integrate_slices(norm01(nd.squeeze(label[-1]))))
sw.add_image('train_predictions', norm01(im), global_step=epoch)
sw_cmd = "cd '%s' \ntensorboard --logdir=./logs --host=127.0.0.1 --port=8888" % opts.dir_out
print('Copy paste this to view Tensorboard:\n%s' % sw_cmd)
if (epoch + 1) % opts.checkpoint_interval == 0:
model.save_params("%sEpoch%03d.params" % (dir_checkpoints, epoch))
if loss_val == lowest_loss:
model.save_params("%sBest_epoch.params" % dir_checkpoints)
print('Training loss: {:.2f}, Validation loss: {:.2f}, '
'\nBest Validation Loss: {:.2f} at epoch {:03d}'
'\nEpoch time: {:.2f} secs'.format(loss_train, loss_val,
lowest_loss, best_epoch,
time.time() - etic))
print('Total training time: {:.2f} secs'.format(time.time() - tic))
X = nd.array([0, 1])
model = MyModel()
batch_size = 8
X = nd.random.normal(scale=1, shape=(100, 2))
X1 = nd.arange(0, 200).reshape(100, 2)
w1 = 1
w2 = 2
Y = w1 * X[:, 0] + w2 * X[:, 1]
dataset = gdata.ArrayDataset(X, Y)
data_iter = gdata.DataLoader(dataset, batch_size, shuffle=True)
for X, y in data_iter:
print(X, y)
break
model.initialize()
trainer = gluon.Trainer(model.collect_params(), 'adam', {'learning_rate': 10})
loss = loss.L2Loss()
for i in range(10):
for X, Y in data_iter:
with autograd.record():
y_pre = model(X)
l = loss(y_pre, Y)
l.backward()
trainer.step(batch_size)
l = loss(model(X), Y)
print('epoch %d, loss: %f' % (i, l.mean().asnumpy()))
print(model.params['mymodel0_line_weight'].data())