def train(num_gpus, batch_size, lr):
train_iter, test_iter = gb.load_data_fashion_mnist(batch_size)
ctx = [mx.gpu(i) for i in range(num_gpus)]
print('training on ', ctx)
net = resnet18(10)
net.initialize(init=init.Normal(sigma=0.01), ctx=ctx)
trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': lr})
for epoch in range(5):
start = time()
for X, y in train_iter:
gpu_Xs = gutils.split_and_load(X, ctx)
gpu_ys = gutils.split_and_load(y, ctx)
with autograd.record():
ls = [
loss(net(gpu_X), gpu_y)
for gpu_X, gpu_y in zip(gpu_Xs, gpu_ys)
]
for l in ls:
l.backward()
trainer.step(batch_size)
nd.waitall()
print('epoch %d, training time: %.1f sec' % (epoch, time() - start))
test_acc = gb.evaluate_accuracy(test_iter, net, ctx[0])
print('validation accuracy %.4f' % (test_acc))
def train(num_gpus, batch_size, lr):
train_iter, test_iter = gb.load_data_fashion_mnist(
batch_size, root="../data/fashion-mnist")
ctx = [mx.gpu(i) for i in range(num_gpus)]
print("running on:", ctx)
net.initialize(init=init.Normal(sigma=0.01), ctx=ctx)
trainer = gluon.Trainer(net.collect_params(), "sgd", {"learning_rate": lr})
loss = gloss.SoftmaxCrossEntropyLoss()
for epoch in range(4):
start = time.time()
for X, y in train_iter:
gpu_Xs = gutils.split_and_load(X, ctx)
gpu_ys = gutils.split_and_load(y, ctx)
with autograd.record():
ls = [
loss(net(gpu_X), gpu_y)
for gpu_X, gpu_y in zip(gpu_Xs, gpu_ys)
]
for l in ls:
l.backward()
trainer.step(batch_size)
nd.waitall()
train_time = time.time() - start
test_acc = gb.evaluate_accuracy(test_iter, net, ctx[0])
print("epoch %d, time: %.1f sec, test acc: %.2f" %
(epoch + 1, train_time, test_acc))
def _get_batch(batch, ctx):
"""Return features and labels on ctx."""
features, labels = batch
if labels.dtype != features.dtype:
labels = labels.astype(features.dtype)
return (gutils.split_and_load(features, ctx),
gutils.split_and_load(labels, ctx), features.shape[0])
def train_batch(self, data, label):
"""
Train batch
:param data:
:param label:
:return:
"""
data_lst = split_and_load(data, self.ctx)
label_lst = split_and_load(label, self.ctx)
criterion_xent = SoftmaxCELoss()
with autograd.record():
losses = []
for x, y in zip(data_lst, label_lst):
y_hat = self.inference(x)
l_xent = criterion_xent(y_hat, y[:, 0])
loss = l_xent
losses.append(loss)
# logging
self.metrics['Train-Xent'].update(None, [l_xent])
self.metrics['Train-Acc'].update([y[:, 0]], [y_hat])
for l in losses:
l.backward()
self.trainer.step(data.shape[0])
self.cur_iter += 1
if self.args.log_itv != 0 and self.cur_iter % self.args.log_itv == 0:
self.log_iter()
def _get_batch(batch, ctx):
features, labels = batch
if labels.dtype != features.dtype:
labels = labels.astype(features.dtype)
# 当 ctx 包含多个 GPU 时，划分⼩批量数据样本并复制到各个 GPU 上。
return (gutils.split_and_load(features, ctx),
gutils.split_and_load(labels, ctx), features.shape[0])
def _get_batch(batch, ctx):
"""Return features and labels on ctx."""
features, labels = batch
if labels.dtype != features.dtype:
labels = labels.astype(features.dtype)
return (gutils.split_and_load(features, ctx),
gutils.split_and_load(labels, ctx), features.shape[0])
def validate_yolov3_coco(net, val_data_loader, ctx, eval_metric):
eval_metric.reset()
net.set_nms(nms_thresh=0.45, nms_topk=400)
mx.nd.waitall()
net.hybrydize()
for batch in val_data_loader:
data = utils.split_and_load(batch[0], ctx_list=ctx, even_split=False)
label = utils.split_and_load(batch[1], ctx_list=ctx, even_split=False)
det_bboxes, det_ids, det_scores = [], [], []
gt_bboxes, gt_ids, gt_difficults = [], [], []
for x, y in zip(data, label):
ids, scores, bboxes = net(x)
det_ids.append(ids)
det_scores.append(scores)
det_bboxes.append(bboxes.clip(0, batch[0].shape[2]))
gt_ids.append(y.slice_axis(axis=-1, begin=4, end=5))
gt_bboxes.append(y.slice_axis(axis=-1, begin=0, end=4))
gt_difficults.append(
y.slice_axis(axis=-1, begin=5, end=6
) if y.shape[-1] > 5 else None)
eval_metric.update(det_bboxes, det_ids, det_scores, gt_bboxes, gt_ids,
gt_difficults)
return eval_metric.get()
def test(test_net, ctx, test_loader, iteration, logger):
# print("Start testing iter %d." % iteration)
Loss = gloss.SoftmaxCrossEntropyLoss()
metric = mx.metric.Accuracy()
metric_top5 = mx.metric.TopKAccuracy(5)
test_loss = mx.metric.Loss()
for batch in test_loader:
trans = gutils.split_and_load(batch[0], ctx)
labels = gutils.split_and_load(batch[1], ctx)
outputs = [test_net(tran) for tran in trans]
losses = [
Loss(output, label) for output, label in zip(outputs, labels)
]
test_loss.update(0, losses)
metric.update(labels, outputs)
metric_top5.update(labels, outputs)
_, test_top1_acc = metric.get()
_, test_top5_acc = metric_top5.get()
_, test_loss = test_loss.get()
if test_top1_acc >= 0.7:
test_net.save_parameters('imagenet_param/test_iter%d_%.5f.param' %
(iteration, test_top1_acc))
test_str = ("test_Loss: %f, test top1-acc %f, test top5-acc %f." %
(test_loss, test_top1_acc, test_top5_acc))
logger.info(test_str)
def train_model():
batch_size = 10
num_epochs = 10
ctx = [mx.gpu(i) for i in range(1)]
net = get_model()
net.initialize(ctx=ctx)
net.collect_params().reset_ctx(ctx)
net.hybridize()
loss = FocalLoss()
trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': learning_rate, 'wd': 0.001})
for epoch in range(1, num_epochs + 1):
for X, y in data_iter(batch_size, data_train_index, data_train_im, data_train_gt, ctx):
# print('x shape is ', X.shape)
# print('y shape is ', y.shape)
gpu_Xs = gutils.split_and_load(X, ctx)
gpu_ys = gutils.split_and_load(y, ctx)
with autograd.record():
ls = [loss(net(gpu_X), gpu_y) for gpu_X, gpu_y in zip(gpu_Xs, gpu_ys)]
for l in ls:
l.backward()
trainer.step(batch_size)
# print(ls[0].asscalar())
print("epoch %d, loss: %f" % (epoch, ls[0].asscalar()))
print("epoch %d, loss: %f" % (epoch, ls[0].asscalar()))
net.save_params('ckp/CSRNet-%d.params' % epoch)
def eval(self, inference, val_loader, log=True, target=True, epoch=True):
"""
Evaluate the model
:param inference: network
:param val_loader: data loader
:param log: log flag
:param target: target flag for updating the record and log
:param epoch: epoch flag for updating the record and log
:return:
"""
mtc_acc = Accuracy()
mtc_acc.reset()
# val_loader.reset()
feature_nest, y_nest, y_hat_nest = [], [], []
for X, Y in val_loader:
X_lst = split_and_load(X, self.args.ctx, even_split=False)
Y_lst = split_and_load(Y, self.args.ctx, even_split=False)
for x, y in zip(X_lst, Y_lst):
y_hat, features = inference(x)
# update metric
mtc_acc.update([y], [y_hat])
y_nest.extend(y.asnumpy())
feature_nest.extend(features.asnumpy())
y_hat_nest.extend(y_hat.asnumpy())
feature_nest = np.array(feature_nest)
y_nest = np.array(y_nest).astype(int)
y_hat_nest = np.array(y_hat_nest)
if log:
target_key = 'Tgt' if target else 'Src'
epoch_key = 'Epoch' if epoch else 'Iter'
record = self.cur_epoch if epoch else self.cur_iter
if mtc_acc.get()[1] > self.records[epoch_key]['%s-Acc' %
target_key]:
if target:
self.records[epoch_key][epoch_key] = record
self.records[epoch_key]['%s-Acc' %
target_key] = mtc_acc.get()[1]
self.records[epoch_key]['%s-label' % target_key] = y_nest
self.records[epoch_key]['%s-preds' % target_key] = y_hat_nest
self.records[epoch_key]['%s-features' %
target_key] = feature_nest
self.save_params(inference, 0, epoch_key)
self.logger.update_scalar(
'%s [%d]: Eval-Acc-%s' % (epoch_key, record, target_key),
mtc_acc.get()[1])
if self.sw:
self.sw.add_scalar('Acc/Eval-%s-Acc-%s' % (epoch, target_key),
mtc_acc.get()[1],
global_step=record)
return mtc_acc.get()[1], y_nest, y_hat_nest, feature_nest
def _get_batch(batch, ctx):
if isinstance(batch, mx.io.DataBatch):
features = batch.data[0]
labels = batch.label[0]
else:
features, labels = batch
return (gutils.split_and_load(features, ctx),
gutils.split_and_load(labels, ctx), features.shape[0])
def train(train_net, iterations, trainer, ctx, lr_period: tuple, lr_decay,
train_loader, test_loader, cat_interval):
# set up logger
logging.basicConfig()
logger = logging.getLogger()
logger.setLevel(logging.INFO)
log_file_path = 'Attention56_train.log'
fh = logging.FileHandler(log_file_path)
logger.addHandler(fh)
net.collect_params().reset_ctx(ctx)
train_gen = inf_train_gen(train_loader)
Loss = gloss.SoftmaxCrossEntropyLoss()
metric = mx.metric.Accuracy()
metric_top5 = mx.metric.TopKAccuracy(5)
train_loss = mx.metric.Loss()
prev_time = datetime.datetime.now()
metric.reset()
train_loss.reset()
for iteration in range(int(iterations)):
batch = next(train_gen)
trans = gutils.split_and_load(batch.data[0], ctx)
labels = gutils.split_and_load(batch.label[0], ctx)
with autograd.record():
outputs = [train_net(tran) for tran in trans]
losses = [
Loss(output, label) for output, label in zip(outputs, labels)
]
for loss in losses:
loss.backward()
trainer.step(batch_size)
train_loss.update(0, losses)
metric.update(labels, outputs)
metric_top5.update(labels, outputs)
if iteration % cat_interval == cat_interval - 1:
cur_time = datetime.datetime.now()
time_str = format_time(prev_time, cur_time)
_, top1_acc = metric.get()
_, top5_acc = metric_top5.get()
_, epoch_loss = train_loss.get()
metric.reset()
metric_top5.reset()
train_loss.reset()
epoch_str = (
"Iter %d. Loss: %.5f, Train top1-acc %f, Train top5-acc %f." %
(iteration, epoch_loss, top1_acc, top5_acc))
prev_time = cur_time
logger.info(epoch_str + time_str + 'lr ' +
str(trainer.learning_rate))
test(train_net, ctx, test_loader, iteration, logger)
if iteration in lr_period:
trainer.set_learning_rate(trainer.learning_rate * lr_decay)
def _get_batch(batch, ctx):
if isinstance(batch, mx.io.DataBatch):
features = batch.data[0]
labels = batch.label[0]
else:
features, labels = batch
return (gutils.split_and_load(features, ctx),
gutils.split_and_load(labels, ctx),
features.shape[0])
def train(transformer, data_iter, lr, num_epochs, vocab, ctx):
print('start training')
print('ctx:', ctx)
trainer = gluon.Trainer(transformer.collect_params(), 'adam', {'learning_rate': lr})
loss = gloss.SoftmaxCrossEntropyLoss()
best_epoch = 0
best_loss = float('Inf')
sw = SummaryWriter(logdir='../logs', flush_secs=5)
for epoch in range(num_epochs):
l_sum = 0.0
for i, data in enumerate(data_iter):
X, Y, label, X_valid_len, Y_valid_len = data
# X = X.as_in_context(ctx)
# Y = Y.as_in_context(ctx)
# label = label.as_in_context(ctx)
gpu_Xs = gutils.split_and_load(X, ctx, even_split=False)
gpu_Ys = gutils.split_and_load(Y, ctx, even_split=False)
gpu_labels = gutils.split_and_load(label, ctx, even_split=False)
with autograd.record():
# l = batch_loss(transformer, X, Y, vocab, loss)
ls = [batch_loss(transformer, gpu_X, gpu_Y, gpu_label, vocab, loss)
for gpu_X, gpu_Y, gpu_label in zip(gpu_Xs, gpu_Ys, gpu_labels)]
# l.backward()
b_loss = 0.0
for l in ls:
l.backward()
b_loss += l.asscalar()
trainer.step(X.shape[0])
nd.waitall()
l_sum += b_loss
if i % 100 == 0:
info = "epoch %d, batch %d, batch_loss %.3f" % (epoch, i, b_loss)
print(info)
sw.add_scalar(tag='batch_loss', value=b_loss, global_step=i)
cur_loss = l_sum / len(data_iter)
# 保存模型
if cur_loss < best_loss:
best_loss = cur_loss
best_epoch = epoch
if not os.path.exists('../model'):
os.mkdir('../model')
transformer.save_parameters('../model/transformer' + str(epoch) + '.params')
info = "epoch %d, loss %.3f, best_loss %.3f, best_epoch %d" % (
epoch, cur_loss, best_loss, best_epoch)
print(info)
sw.add_scalar(tag='loss', value=cur_loss, global_step=epoch)
def evaluate_confusion_matrix(data_iterator, net, ctx=ctx):
import numpy as np
cm = np.zeros([classes, classes])
for i, (X, y) in enumerate(data_iterator):
X = X / 255
data = gutils.split_and_load(X, ctx, even_split=False)
label = gutils.split_and_load(y, ctx, even_split=False)
output = [(net(data), label) for data, label in zip(data, label)]
for result in output:
cm += ConfusionMatrix(result[0], result[1])
return cm
def _eval(self):
n_correct = 0
edit_dis = 0
for images, labels in tqdm(self.validate_loader, desc='test model'):
gpu_images = gutils.split_and_load(images, self.ctx)
gpu_labels = gutils.split_and_load(labels, self.ctx)
preds = [self.model(x)[0] for x in gpu_images]
batch_dict = self.accuracy_batch(preds, gpu_labels, phase='VAL')
n_correct += batch_dict['n_correct']
edit_dis += batch_dict['edit_dis']
return {'n_correct': n_correct, 'edit_dis': edit_dis}
def _get_batch(batch, ctx):
"""Return features and labels on ctx."""
if isinstance(batch, mx.io.DataBatch):
features = batch.data[0]
labels = batch.label[0]
else:
features, labels = batch
if labels.dtype != features.dtype:
labels = labels.astype(features.dtype)
return (gutils.split_and_load(features, ctx),
gutils.split_and_load(labels, ctx), features.shape[0])
def simulate_z_posterior_mcmc(self,
x,
nsim,
init=None,
burnin=100,
step_size=0.1):
# This function relies on the hybridized self.langevin, so we need to make sure
# nsim is equal to self.langevin.nchain. If not, we reinitialize self.langevin
if nsim != self.langevin.nchain:
self.langevin = LangevinLogPDF(self.dec, nsim, self.log_cond_pdf)
self.langevin_loss = LangevinLoss(self.dec, nsim,
self.log_cond_pdf)
self.langevin.hybridize()
self.langevin_loss.hybridize()
n = x.shape[0]
# For now neural network parameters do not need gradient
for name, param in self.loglik_params.items():
param.grad_req = "null"
if init is None:
mu, logvar = self.enc(x.as_in_context(self.ctx[0]))
init, _ = self.simulate_latent(mu, logvar, nsim, F=nd)
init = init.reshape((nsim, n, self.latent_dim))
# init [nsim x n x d], x [n x d]
# split_and_load() works on the first axis, first transpose init, and then transform back
x_gpus = utils.split_and_load(x, self.ctx)
init_gpus = utils.split_and_load(init.transpose((1, 0, 2)), self.ctx)
chains_gpus = []
for x, init in zip(x_gpus, init_gpus):
# Transpose back
init = init.transpose((1, 0, 2))
sampler = LangevinMultiChain(shape=init.shape[1:],
nchain=nsim,
ctx=x.context)
chains = sampler.sample(model=self,
start=init,
step_size=step_size,
burnin=burnin,
args={"x": x})
chains_gpus.append(chains)
nd.waitall()
chains = [c.as_in_context(x.context) for c in chains_gpus]
# Restore gradient flag
for name, param in self.loglik_params.items():
# Parameters that have no gradient, e.g. in batch normalization,
# are unaffected
param.grad_req = "write"
return nd.concat(*chains, dim=1)
def evaluate_accuracy(data_iterator, net, ctx=ctx):
acc = 0.
for i, (X, y) in enumerate(data_iterator):
X = X / 255
data = gutils.split_and_load(X, ctx, even_split=False)
label = gutils.split_and_load(y, ctx, even_split=False)
output = [(net(data), label) for data, label in zip(data, label)]
for result in output:
acc += accuracy(result[0], result[1])
# if i + 1 >= 30 :
# return acc / (30 * len(ctx))
return acc / (len(data_iterator) * len(ctx))
def batch_fn(batch, ctx):
# data = split_and_load(batch[0], ctx_list=ctx, batch_axis=0)
# data2 = split_and_load(batch[1], ctx_list=ctx, batch_axis=0)
# label = split_and_load(batch[2], ctx_list=ctx, batch_axis=0)
# return data, data2, label
# data = split_and_load(batch[0], ctx_list=ctx, batch_axis=0)
# label = split_and_load(batch[1], ctx_list=ctx, batch_axis=0)
# return data, label
data = split_and_load(batch[0][0], ctx_list=ctx, batch_axis=0)
data2 = split_and_load(batch[0][1], ctx_list=ctx, batch_axis=0)
label = split_and_load(batch[1], ctx_list=ctx, batch_axis=0)
return data, data2, label
def _get_batch(batch, ctx):
"""return features and labels on ctx"""
if isinstance(batch, mx.io.DataBatch):
features = batch.data[0]
labels = batch.label[0]
else:
features, labels = batch
if labels.dtype != features.dtype:
labels = labels.astype(features.dtype)
return (gutils.split_and_load(features, ctx),
gutils.split_and_load(labels, ctx),
features.shape[0])
def unpack_batch(batch, ctx_list):
data = split_and_load(batch.data[0],
ctx_list=ctx_list,
batch_axis=0,
even_split=False)
label = split_and_load(batch.label[0],
ctx_list=ctx_list,
batch_axis=0,
even_split=False)
if batch.index is None:
return data, label
else:
return data, label, batch.index[0]
def _get_batch(batch, ctx):
if isinstance(batch, mx.io.DataBatch): # in fact, the batch is a list
features = batch.data[0]
labels = batch.label[0]
else:
features, labels = batch
# use split_and_load: split the data(ndarray) to len(ctx_list) slices along batch_axis, and loads each slice to one context in ctx_list
# here, the len(ctx_list) = 0, return data.as_in_context(ctx), this can be found in the source of this function.
if isinstance(ctx, mx.Context):
ctx = [ctx]
return (gutils.split_and_load(features, ctx),
gutils.split_and_load(labels, ctx),
features.shape[0]) # return the batch size
def get_batch_max_length(self, img_h, img_w, ctx):
from mxnet.gluon import utils as gutils
input = gutils.split_and_load(nd.zeros((2, 3, img_h, img_w)), ctx)
# 特征提取阶段
visual_feature = [self.feature_extraction(x) for x in input]
self.batch_max_length = visual_feature[0].shape[-1]
return self.batch_max_length
def multi_gpus_forward(net, ctx, img, labels=None, bboxes=None):
img_list = gutils.split_and_load(img, ctx)
if autograd.is_training():
labels_list = gutils.split_and_load(labels, ctx)
bboxes_list = gutils.split_and_load(bboxes, ctx)
rpn_cls_losses = []
rpn_box_losses = []
rcnn_cls_losses = []
rcnn_box_losses = []
total_losses = []
for img, labels, bboxes in zip(img_list, labels_list, bboxes_list):
rpn_cls_loss, rpn_box_loss, rcnn_cls_loss, rcnn_box_loss, total_loss = net(img, labels, bboxes)
rpn_cls_losses.append(rpn_cls_loss)
rpn_box_losses.append(rpn_box_loss)
rcnn_cls_losses.append(rcnn_cls_loss)
rcnn_box_losses.append(rcnn_box_loss)
total_losses.append(total_loss)
rpn_cls_loss = np.mean([loss.asscalar() for loss in rpn_cls_losses])
rpn_box_loss = np.mean([loss.asscalar() for loss in rpn_box_losses])
rcnn_cls_loss = np.mean([loss.asscalar() for loss in rpn_cls_losses])
rcnn_box_loss = np.mean([loss.asscalar() for loss in rcnn_cls_losses])
total_loss = np.mean([loss.asscalar() for loss in total_losses])
return rpn_cls_loss, rpn_box_loss, rcnn_cls_loss, rcnn_box_loss, total_loss
else:
cls_ids_list = []
scores_list = []
bboxes_list = []
for img in img_list:
cls_ids, scores, bboxes = net(img)
cls_ids_list.append(cls_ids)
scores_list.append(scores)
bboxes_list.append(bboxes)
cls_ids_list = [item.asnumpy() for item in cls_ids_list]
scores_list = [item.asnumpy() for item in scores_list]
bboxes_list = [item.asnumpy() for item in bboxes_list]
cls_ids = np.concatenate(cls_ids_list)
scores = np.concatenate(scores_list)
bboxes = np.concatenate(bboxes_list)
return cls_ids, scores, bboxes
def split_map(obj):
if isinstance(obj, NDArray):
return split_and_load(obj, ctx_list, batch_axis, even_split=False)
if isinstance(obj, tuple) and len(obj) > 0:
return list(zip(*map(split_map, obj)))
if isinstance(obj, list) and len(obj) > 0:
return list(map(list, zip(*map(split_map, obj))))
if isinstance(obj, dict) and len(obj) > 0:
return list(map(type(obj), zip(*map(split_map, obj.items()))))
return [obj for _ in ctx_list]
def train_epoch(self, inference, trainer, **kwargs):
"""
Training with dSNEt loss
:param inference: inference
:param trainer: trainer of inference
:return:
"""
for Xs, Ys, Xt, Yt, _ in self.train_src_loader:
Xs_lst = split_and_load(Xs, self.args.ctx, even_split=False)
Ys_lst = split_and_load(Ys, self.args.ctx, even_split=False)
Xt_lst = split_and_load(Xt, self.args.ctx, even_split=False)
Yt_lst = split_and_load(Yt, self.args.ctx, even_split=False)
if self.args.train_src:
self.train_batch(Xs_lst,
Ys_lst,
Xt_lst,
Yt_lst,
inference,
target=False)
trainer.step(Xs.shape[0])
self.train_batch(Xt_lst,
Yt_lst,
Xs_lst,
Ys_lst,
inference,
target=True)
trainer.step(Xt.shape[0])
if self.args.log_itv > 0 and self.cur_iter % self.args.log_itv == 0:
self.log_iter()
if self.args.eval:
self.eval(inference,
self.test_tgt_loader,
target=True,
epoch=False)
self.log_epoch()
if self.args.eval and self.cur_epoch > self.args.eval_epoch:
self.eval(inference, self.test_tgt_loader, target=True, epoch=True)
def val_net(self, net, val_data, len_vd):
n_batch = int(len_vd / self.batch_size)
self.print_info('训练 - 样本数:{}, 批次样本: {}, 批次数: {}'.format(
len_vd, self.batch_size, n_batch))
e_r, e_p, e_f1 = 0, 0, 0
for i, batch in enumerate(val_data):
data, labels = batch[0], batch[1].astype('float32')
data = split_and_load(data,
ctx_list=self.ctx,
batch_axis=0,
even_split=False) # 多GPU
labels = split_and_load(labels,
ctx_list=self.ctx,
batch_axis=0,
even_split=False)
outputs = [net(X) for X in data]
br, bp, bf1 = self.get_batch_rpf(outputs, labels)
e_r += br
e_p += bp
e_f1 += bf1
self.print_info(
'validation: batch: {}, recall: {:.2f}, precision: {:.2f}, f1: {:.2f}'
.format(i, br, bp, bf1))
n_batch = i + 1
e_r /= n_batch
e_p /= n_batch
e_f1 /= n_batch
self.print_info(
'validation: recall: {:.2f}, precision: {:.2f}, f1: {:.2f}'.format(
e_r, e_p, e_f1))
return e_r, e_p, e_f1
def bias_correction(self, x, init=None, burnin=100, step_size=0.1):
postz = self.simulate_z_posterior_mcmc(x,
nsim=self.nchain,
init=init,
burnin=burnin,
step_size=step_size)
# z [nsim x n x d], x [n x d]
# split_and_load() works on the first axis, first transpose z, and then transform back
x_gpus = utils.split_and_load(x, self.ctx)
z_gpus = utils.split_and_load(postz.transpose((1, 0, 2)), self.ctx)
loss_gpus = []
with autograd.record():
for x, z in zip(x_gpus, z_gpus):
# Transpose back
z = z.transpose((1, 0, 2))
loss = self.langevin_loss(z.reshape((-1, self.latent_dim)), x)
loss_gpus.append(loss)
return loss_gpus, postz
def extract_features(self, preprocessed_data, verbose=True):
"""
Parameters
----------
preprocessed_data : SArray
Returns
-------
numpy array containing the deep features
"""
import numpy as np
last_progress_update = _time.time()
if _mac_ver() < (10, 14):
# Use MXNet
preprocessed_data = mx.nd.array(preprocessed_data)
ctx_list = self.ctx
if len(preprocessed_data) < len(ctx_list):
ctx_list = ctx_list[:len(preprocessed_data)]
batches = utils.split_and_load(preprocessed_data,
ctx_list=ctx_list,
even_split=False)
deep_features = []
for i, cur_batch in enumerate(batches):
y = self.vggish_model.forward(cur_batch).asnumpy()
for i in y:
deep_features.append(i)
# If `verbose` is set, print an progress update about every 20s
if verbose and _time.time() - last_progress_update >= 20:
print("Extracted {} of {} batches".format(
i, len(audio_data)))
last_progress_update = _time.time()
else:
# Use Core ML
deep_features = []
for i, cur_example in enumerate(preprocessed_data):
for cur_frame in cur_example:
x = {'input1': [cur_frame]}
y = self.vggish_model.predict(x)
deep_features.append(y['output1'])
# If `verbose` is set, print an progress update about every 20s
if verbose and _time.time() - last_progress_update >= 20:
print("Extracted {} of {}".format(i,
len(preprocessed_data)))
last_progress_update = _time.time()
return np.asarray(deep_features)
def evaluate_net(self, base_net, val_data):
triplet_loss = gluon.loss.TripletLoss(margin=0)
rate = 0.0
sum_correct, sum_all = 0, 0
for i, batch in enumerate(val_data):
data, labels = batch[0], batch[1].astype('float32')
data = split_and_load(data,
ctx_list=self.ctx,
batch_axis=0,
even_split=False) # 多GPU
labels = split_and_load(labels,
ctx_list=self.ctx,
batch_axis=0,
even_split=False)
for X in data:
anchor_ins, pos_ins, neg_ins = [], [], []
for b_X in X:
anchor_ins.append(nd.expand_dims(b_X[0], axis=0))
pos_ins.append(nd.expand_dims(b_X[1], axis=0))
neg_ins.append(nd.expand_dims(b_X[2], axis=0))
anchor_ins = nd.concatenate(anchor_ins, axis=0)
pos_ins = nd.concatenate(pos_ins, axis=0)
neg_ins = nd.concatenate(neg_ins, axis=0)
inter1 = base_net(anchor_ins)
inter2 = base_net(pos_ins)
inter3 = base_net(neg_ins)
loss = triplet_loss(inter1, inter2, inter3) # TripletLoss
n_correct = np.sum(np.where(loss == 0, 1, 0))
sum_all += loss.shape[0]
sum_correct += n_correct
rate = safe_div(sum_correct, sum_all)
self.print_info('验证Batch: {}, 准确率: {:.4f} ({} / {})'.format(
i, rate, sum_correct, sum_all))
rate = safe_div(sum_correct, sum_all)
self.print_info('验证准确率: %.4f (%s / %s)' % (rate, sum_correct, sum_all))
return rate
def predict():
import numpy as np
from scipy import misc
# predict_data = gluon.data.vision.datasets.ImageFolderDataset('predict_img').transform_first(val_transform)
# predict_iterator = gluon.data.DataLoader(predict_data, batch_size=batch_size, num_workers=4)
for item, (X, y) in enumerate(train_loader):
logging.info(len(train_loader))
logging.info(item)
X = X / 255
data = gutils.split_and_load(X, ctx, even_split=False)
label = gutils.split_and_load(y, ctx, even_split=False)
output = [mx.ndarray.SoftmaxActivation(data=net(i)) for i in data]
for i in range(len(output)):
y_pred = output[i].asnumpy().tolist()
y_pred_index = np.argmax(y_pred, axis=1)
y_pred_prob = np.max(y_pred, axis=1)
y_label = label[i].asnumpy().tolist()
y_label = np.array(y_label)
error_index = [
i for i in range(len(y_pred_index))
if y_pred_index[i] != y_label[i]
]
if len(error_index):
for j in error_index:
predict_name = name_list[int(y_pred_index.tolist()[j])]
label_name = name_list[int(y_label.tolist()[j])]
predict_prob = str(y_pred_prob.tolist()[j])
img_array = data[i][j] * 255
misc.imsave(
'img_file/hard_train/' + label_name + '/' +
# str(item) + '_' + str(i) + '_' + str(j) + '_' +
predict_prob + '-' + label_name + '-->' +
predict_name + '.bmp',
img_array.asnumpy().transpose(1, 2, 0))
return 0
def train(num_gpus, batch_size, lr):
comm = MPI.COMM_WORLD
comm_rank = comm.Get_rank()
comm_size = comm.Get_size()
train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)
#ctx = [mx.gpu(i) for i in range(num_gpus)]
if comm_rank == 0:
ctx = mx.gpu(0)
else:
ctx = mx.gpu(1)
print('running on:', ctx)
net.initialize(init=init.Normal(sigma=0.01), ctx=ctx, force_reinit=True)
trainer = gluon.Trainer(net.collect_params(),
'sgd', {'learning_rate': lr},
SSP_FLAG=True,
thre=2)
loss = gloss.SoftmaxCrossEntropyLoss()
for epoch in range(400000):
start = time.time()
for X, y in train_iter:
gpu_Xs = gutils.split_and_load(X, ctx)
gpu_ys = gutils.split_and_load(y, ctx)
with autograd.record():
ls = [
loss(net(gpu_X), gpu_y)
for gpu_X, gpu_y in zip(gpu_Xs, gpu_ys)
]
for l in ls:
l.backward()
trainer.step(epoch, batch_size)
train_time = time.time() - start
test_acc = d2l.evaluate_accuracy(test_iter, net, ctx[comm_rank])
print('epoch %d, time %.1f sec, test acc %.2f, process %d' %
(epoch + 1, train_time, test_acc, comm_rank))
def get_classifer(max_sample_num):
ctx = mx.gpu()
crop_hw = (config.height, config.width)
ds_train = JSDataset(config.train_root, fortrain=True, crop_hw = crop_hw,max_sample_num = max_sample_num)
ds_test = JSDataset(config.test_root, fortrain=False, crop_hw=crop_hw, max_sample_num = max_sample_num)
trainiter = gluon.data.DataLoader(ds_train,batch_size=config.batch_size, shuffle=True,last_batch="rollover")
testiter = gluon.data.DataLoader(ds_test,batch_size=config.batch_size, shuffle=False,last_batch="rollover")
logging.info("train num: {} test num: {}".format(len(ds_train), len(ds_test)))
max_update = config.max_epoch * len(ds_train) // config.batch_size
lr_sch = mx.lr_scheduler.PolyScheduler(max_update=max_update,base_lr=config.base_lr,pwr=1)
net = CIFARNET_QUICK(class_num = config.class_num, ctx=ctx)
if not (config.pretrained_model is None) and not (config.pretrained_model == ""):
net.load_params(config.pretrained_model,ctx = ctx)
logging.info("loading model {}".format(config.pretrained_model))
trainer = mx.gluon.Trainer(net.collect_params(),optimizer=config.optimizer,optimizer_params={"learning_rate":config.base_lr})
loss_ce = mx.gluon.loss.SoftmaxCrossEntropyLoss()
class ACC_SHOW(object):
def __init__(self,label_num):
self.label_num = label_num
self.axis = 1
self.acc = {'total':0,'hit':0}
self.acc_per_class = OrderedDict()
for key in range(label_num):
self.acc_per_class[key] = {'total':0,'hit':0}
return
def reset(self):
self.acc = {'total':0,'hit':0}
self.acc_per_class = OrderedDict()
for key in range(self.label_num):
self.acc_per_class[key] = {'total':0,'hit':0}
return
def update(self,preds, labels):
if isinstance(preds[0],mx.nd.NDArray):
preds = map(lambda pred: pred.asnumpy(),preds)
labels = map(lambda label: label.asnumpy(),labels)
for pred, label in zip(preds,labels):
pred_label = np.argmax(pred,axis=self.axis)
label = label.flatten().astype('int32')
pred_label = pred_label.flatten().astype('int32')
for p,l in zip(pred_label,label):
self.acc_per_class[l]['total'] += 1
self.acc['total'] += 1
if l == p:
self.acc_per_class[l]['hit'] += 1
self.acc['hit'] += 1
return
def _calc_acc(self,md):
total = md['total']
hit = md['hit']
if total < 1:
return 0
return float(hit) / total
def get_acc(self):
return self._calc_acc(self.acc)
def get(self):
infos = ['acc {:.5} acc_per_class'.format( self._calc_acc(self.acc) )]
for key in self.acc_per_class.keys():
#print self.acc_per_class[key]
infos.append('{:.3}'.format(self._calc_acc(self.acc_per_class[key])))
return ' '.join(infos)
class LOSS_SHOW(object):
def __init__(self):
self.loss_list = []
def reset(self):
self.loss_list = []
def update(self, loss_list):
if isinstance(loss_list[0],mx.nd.NDArray):
loss_list = map(lambda loss: loss.asnumpy(), loss_list)
loss = np.vstack(loss_list)
#print loss.tolist()[0]
self.loss_list.extend(loss.tolist()[0])
def get(self):
return "loss {:.5}".format( np.asarray(self.loss_list).mean() )
import pdb
def show_gradient(net):
return
grads_list = []
for block in net.layers:
if not isinstance(block, CIFARNET_BLOCK) and not isinstance(block, CIFARNET_BLOCK_A):
continue
for layer in block.layers:
if not isinstance(layer, gluon.nn.Conv2D):
continue
grads = layer.weight.grad().as_in_context(mx.cpu()).asnumpy()
grads_list.append(grads.mean())
grads_list.append(grads.max())
grads_list.append(grads.min())
line = ['grads: ']
for grads in grads_list:
line.append( '%.6f'%grads )
logging.info(','.join(line))
return
class TopAcc:
def __init__(self):
self.path = ""
self.score = 0
def update(self, path, score):
if self.score < score:
self.score = score
self.path = path
return
def get_top(self):
return self.path,self.score
top_acc = TopAcc()
loss_show = LOSS_SHOW()
acc = ACC_SHOW( config.class_num )
display_iter = len(ds_train) // (2 * config.batch_size )
if display_iter < 1:
display_iter = 1
update = 0
for epoch in range(config.max_epoch):
acc.reset()
loss_show.reset()
for batch in trainiter:
update += 1
data, label = batch
data_list, label_list = utils.split_and_load(data,ctx_list=[ctx]), utils.split_and_load(label,ctx_list=[ctx])
with mx.autograd.record():
pred_list = map(lambda data: net(data), data_list)
loss_list = map(lambda (pred,label): loss_ce(pred,label), zip(pred_list,label_list))
for loss in loss_list:
loss.backward()
trainer.step(config.batch_size)
mx.nd.waitall()
acc.update(labels = label_list,preds = pred_list)
loss_show.update(loss_list)
if 0 == (update % display_iter):
logging.info("train update {} lr {} {} {}".format(update,trainer.learning_rate,loss_show.get(), acc.get()))
trainer.set_learning_rate(lr_sch(update))
acc.reset()
show_gradient(net)
loss_show.reset()
for (data,label) in testiter:
data_list,label_list = utils.split_and_load(data,[ctx]),utils.split_and_load(label, [ctx])
pred_list = map(lambda data : net(data), data_list)
loss_list = map(lambda (pred,label): loss_ce(pred,label), zip(pred_list,label_list))
mx.nd.waitall()
acc.update(labels = label_list, preds = pred_list)
loss_show.update(loss_list)
logging.info("test update {} epoch {} {} {}".format(update,epoch,loss_show.get(), acc.get()))
if epoch % config.save_epoch_step == 0:
net.save_params(config.model_path(epoch))
top_acc.update( config.model_path(epoch), acc.get_acc() )
net.save_params(config.model_path("last"))
return top_acc.get_top()
def _check_batchnorm_result(input, num_devices=1, cuda=False):
from mxnet.gluon.utils import split_and_load
def _find_bn(module):
if isinstance(module, (mx.gluon.nn.BatchNorm, mx.gluon.contrib.nn.SyncBatchNorm)):
return module
elif isinstance(module.module, (mx.gluon.nn.BatchNorm, mx.gluon.contrib.nn.SyncBatchNorm)):
return module.module
raise RuntimeError('BN not found')
def _syncParameters(bn1, bn2, ctx):
ctx = input.context
bn2.gamma.set_data(bn1.gamma.data(ctx))
bn2.beta.set_data(bn1.beta.data(ctx))
bn2.running_mean.set_data(bn1.running_mean.data(ctx))
bn2.running_var.set_data(bn1.running_var.data(ctx))
input1 = input.copy()
input2 = input.copy()
if cuda:
input1 = input.as_in_context(mx.gpu(0))
ctx_list = [mx.gpu(i) for i in range(num_devices)]
else:
ctx_list = [mx.cpu(0) for _ in range(num_devices)]
nch = input.shape[1]
bn1 = mx.gluon.nn.BatchNorm(in_channels=nch)
bn2 = mx.gluon.contrib.nn.SyncBatchNorm(in_channels=nch, num_devices=num_devices)
bn1.initialize(ctx=ctx_list[0])
bn2.initialize(ctx=ctx_list)
# using the same values for gamma and beta
#_syncParameters(_find_bn(bn1), _find_bn(bn2), ctx_list[0])
input1.attach_grad()
inputs2 = split_and_load(input2, ctx_list, batch_axis=0)
for xi in inputs2:
xi.attach_grad()
with mx.autograd.record():
output1 = bn1(input1)
output2 = [bn2(xi) for xi in inputs2]
loss1 = (output1 ** 2).sum()
loss2 = [(output ** 2).sum() for output in output2]
mx.autograd.backward(loss1)
mx.autograd.backward(loss2)
output2 = mx.nd.concat(*[output.as_in_context(input.context) for output in output2], dim=0)
# assert forwarding
assert_almost_equal(input1.asnumpy(), input2.asnumpy(), atol=1e-3, rtol=1e-3)
assert_almost_equal(output1.asnumpy(), output2.asnumpy(), atol=1e-3, rtol=1e-3)
assert_almost_equal(_find_bn(bn1).running_mean.data(ctx_list[0]).asnumpy(),
_find_bn(bn2).running_mean.data(ctx_list[0]).asnumpy(),
atol=1e-3, rtol=1e-3)
assert_almost_equal(_find_bn(bn1).running_var.data(ctx_list[0]).asnumpy(),
_find_bn(bn2).running_var.data(ctx_list[0]).asnumpy(),
atol=1e-3, rtol=1e-3)
input2grad = mx.nd.concat(*[output.grad.as_in_context(input.context) for output in inputs2], dim=0)
assert_almost_equal(input1.grad.asnumpy(), input2grad.asnumpy(), atol=1e-3, rtol=1e-3)
