def train(train_net, iterations, trainer, ctx, lr_period: tuple, lr_decay,
train_loader, test_loader, cat_interval):
# set up logger
logging.basicConfig()
logger = logging.getLogger()
logger.setLevel(logging.INFO)
log_file_path = 'Attention56_train.log'
fh = logging.FileHandler(log_file_path)
logger.addHandler(fh)
net.collect_params().reset_ctx(ctx)
train_gen = inf_train_gen(train_loader)
Loss = gloss.SoftmaxCrossEntropyLoss()
metric = mx.metric.Accuracy()
metric_top5 = mx.metric.TopKAccuracy(5)
train_loss = mx.metric.Loss()
prev_time = datetime.datetime.now()
metric.reset()
train_loss.reset()
for iteration in range(int(iterations)):
batch = next(train_gen)
trans = gutils.split_and_load(batch.data[0], ctx)
labels = gutils.split_and_load(batch.label[0], ctx)
with autograd.record():
outputs = [train_net(tran) for tran in trans]
losses = [
Loss(output, label) for output, label in zip(outputs, labels)
]
for loss in losses:
loss.backward()
trainer.step(batch_size)
train_loss.update(0, losses)
metric.update(labels, outputs)
metric_top5.update(labels, outputs)
if iteration % cat_interval == cat_interval - 1:
cur_time = datetime.datetime.now()
time_str = format_time(prev_time, cur_time)
_, top1_acc = metric.get()
_, top5_acc = metric_top5.get()
_, epoch_loss = train_loss.get()
metric.reset()
metric_top5.reset()
train_loss.reset()
epoch_str = (
"Iter %d. Loss: %.5f, Train top1-acc %f, Train top5-acc %f." %
(iteration, epoch_loss, top1_acc, top5_acc))
prev_time = cur_time
logger.info(epoch_str + time_str + 'lr ' +
str(trainer.learning_rate))
test(train_net, ctx, test_loader, iteration, logger)
if iteration in lr_period:
trainer.set_learning_rate(trainer.learning_rate * lr_decay)
def train(train_net, iterations, lr, wd, ctx, lr_period: tuple, lr_decay,
train_loader, test_loader, cat_interval):
net.collect_params().reset_ctx(ctx)
trainer = gluon.Trainer(train_net.collect_params(), 'sgd', {
'learning_rate': lr,
'momentum': 0.9,
'wd': wd
})
# trainer = gluon.Trainer(train_net.collect_params(),
# 'rmsprop', {'learning_rate': lr, 'wd': wd})
train_gen = inf_train_gen(train_loader)
Loss = gloss.SoftmaxCrossEntropyLoss()
metric = mx.metric.Accuracy()
train_loss = mx.metric.Loss()
prev_time = datetime.datetime.now()
metric.reset()
train_loss.reset()
for iteration in range(int(iterations)):
batch = next(train_gen)
trans = batch[0].as_in_context(ctx)
labels = batch[1].as_in_context(ctx)
with autograd.record():
output = train_net(trans)
loss = Loss(output, labels)
loss.backward()
trainer.step(batch_size)
train_loss.update(0, loss)
metric.update(labels, output)
if iteration % cat_interval == cat_interval - 1:
cur_time = datetime.datetime.now()
time_str = format_time(prev_time, cur_time)
_, train_acc = metric.get()
_, epoch_loss = train_loss.get()
epoch_str = ("Iter %d. Loss: %.5f, Train acc %f." %
(iteration, epoch_loss, train_acc))
prev_time = cur_time
print("\033[32m" + epoch_str + time_str + 'lr ' +
str(trainer.learning_rate) + "\033[0m")
test(train_net, ctx, test_loader, iteration)
if iteration in lr_period:
trainer.set_learning_rate(trainer.learning_rate * lr_decay)
def train_mixup(train_net, iterations, lr, wd, ctx, lr_period: tuple, lr_decay,
train_loader, test_loader, cat_interval, logger):
train_net.collect_params().reset_ctx(ctx)
trainer = gluon.Trainer(train_net.collect_params(), 'sgd', {
'learning_rate': lr,
'momentum': 0.9,
'wd': wd
})
train_gen = inf_train_gen(train_loader)
Loss = gluon.loss.SoftmaxCrossEntropyLoss(sparse_label=False)
metric = mx.metric.RMSE()
train_loss = mx.metric.Loss()
prev_time = datetime.datetime.now()
alpha = 1
classes = 10
metric.reset()
train_loss.reset()
print("Start training with mixup.")
for iteration in range(int(iterations)):
lam = np.random.beta(alpha, alpha)
if iteration >= iterations * 0.8:
lam = 1
batch = next(train_gen)
data = batch[0].as_in_context(ctx)
label = batch[1].as_in_context(ctx)
# trans = [lam * X + (1 - lam) * X[::-1] for X in data]
trans = lam * data + (1 - lam) * data[::-1]
y1 = label_transform(label, classes)
y2 = label_transform(label[::-1], classes)
labels = lam * y1 + (1 - lam) * y2
# for Y in label:
# y1 = label_transform(Y, classes)
# y2 = label_transform(Y[::-1], classes)
# label.append(lam * y1 + (1 - lam) * y2)
with autograd.record():
output = train_net(trans)
loss = Loss(output, labels)
loss.backward()
trainer.step(batch_size)
train_loss.update(0, loss)
output_softmax = nd.SoftmaxActivation(output)
metric.update(labels, output_softmax)
if iteration % cat_interval == cat_interval - 1:
cur_time = datetime.datetime.now()
time_str = format_time(prev_time, cur_time)
_, train_acc = metric.get()
_, epoch_loss = train_loss.get()
epoch_str = ("Iter %d. Loss: %.5f, Train RMSE %.5f." %
(iteration, epoch_loss, train_acc))
prev_time = cur_time
logger.info(epoch_str + time_str + 'lr ' +
str(trainer.learning_rate))
test(train_net, ctx, test_loader, iteration, logger)
if iteration in lr_period:
trainer.set_learning_rate(trainer.learning_rate * lr_decay)