def run_epoch(epoch, data_iter, model, trainer, loss_fn, ctx = mx.cpu()): "Standard Training and Logging Function" start = time.time() total_tokens = 0 tokens = 0 total_loss = 0 for i, batch in enumerate(data_iter): src = batch.src.as_in_context(ctx) trg = batch.trg.as_in_context(ctx) src_mask = batch.src_mask.as_in_context(ctx) trg_mask = batch.trg_mask.as_in_context(ctx) trg_y = batch.trg_y.as_in_context(ctx) ntokens = batch.ntokens with autograd.record(): out = model(src, trg, src_mask, trg_mask) _out = out.reshape(-1, out.shape[-1]) _cols = list(batch.trg_y.reshape(-1).asnumpy()) _rows = list(range(_out.shape[0])) _idx = nd.array([_rows, _cols], ctx = ctx) _trg = nd.scatter_nd(nd.ones_like(trg_y.reshape(-1)), _idx, _out.shape) loss = nd.sum(loss_fn(_out, _trg)) loss.backward() trainer.step(out.shape[0]) total_loss += loss.asnumpy()[0] total_tokens += ntokens.asnumpy()[0] tokens += ntokens.asnumpy()[0] if i % 50 == 0: elapsed = time.time() - start logger.info("Epoch Step: %d Loss: %f Tokens per Sec: %f" % (epoch, loss.asnumpy()[0] / ntokens.asnumpy()[0], tokens / elapsed)) start = time.time() tokens = 0 return total_loss #/ total_tokens
def generate_backgrad_data(net, data, label, ctx=mx.gpu(0), bn_control=None, max_iters=60, sgd=SGD(lr=0.1), # optimizer args iter_log_period=None, show_clip=False, record_detail=False, logger=None,# log args labelmap=None, labeloutputmap=None, loss_f=SCELoss(), # loss_args threshold=None ):#constraint_agrs """ param: net: base net model data: data will be changed.recomand data context in cpu. label: data's label, recomand label context in cpu. ctx: backgrad context, if ctx is gpu and data/label in gpu(or said they in same), then backgradwill change the data iteself, max_iters: max_iters for backgrad lr: lr for backgrad iter_log_period: output log period, None means never output. show_clip: log show backgrad image is clip? loss_f: loss function for backgrad bn_control: bn_control be fore backgrad, None means never use BN Control. sgd: backgrad optimizer method. trheshold: returned data's color clip trheshold. labelmap, labeloutputmap: decide use what label to backgrad generate adversal smaple. use -loss when all None. only one canbe specified. data is better in cpu, if data in ctx(global var), the returned backgrad_data is shallow copy of data. """ if bn_control is not None: bn_control.store() sparse_label = label.copy() if labelmap is not None: label = labelmap(sparse_label) data, label, sparse_label_ctx = data.as_in_context(ctx), label.as_in_context(ctx), sparse_label.as_in_context(ctx) if logger is None: logger = LogRecorder() if record_detail: origin_data = data.copy() constraint = Constraint(threshold) for iters in range(1, max_iters+1): with autograd.record(): data.attach_grad() output = net(data) if labeloutputmap is not None: label = labeloutputmap(output, sparse_label).as_in_context(ctx) loss = loss_f(output, label) if labeloutputmap is None and labelmap is None: loss = -loss loss.backward() mean_loss = nd.mean(loss).asscalar() # reduce will make memory release if record_detail: logger(output, sparse_label_ctx, origin_data, data, loss) logger.print_log(iter_log_period, iters, mean_loss, record_detail, data, show_clip) sgd(data) constraint(data, iters, max_iters) if bn_control is not None: bn_control.load() logger.asnumpy() sgd.clear() return data, (logger, )
def batch_estimate_and_update(self, data: np.ndarray, targets: np.ndarray) -> np.ndarray: input_tensor = mx.nd.array(data) target_tensor = mx.nd.array(targets) with mx.autograd.record(): result = self.model(input_tensor) loss = self.loss_fn(result, target_tensor) loss.backward() self.trainer.step(input_tensor.shape[0]) return result.asnumpy()
def train(self): model = self.getModel() if not os.path.exists(model): os.mkdir(model) ctx = mx.gpu(self.gpu) net, myLoss = self.getNet(ctx) trainI, testI = self.getData(ctx) metric, monitor = self.getMetric() trainer = self.getTrainer(net.collect_params(), trainI.iters) logging.info('') result, epochs = 0, self.getEpoch() for epoch in xrange(1, epochs + 1): logging.info('Epoch[%04d] start ...' % epoch) map(lambda x: x.reset(), [trainI, metric]) for data, label in trainI: with autograd.record(): out = net.forward(data) self.forDebug(out) loss = myLoss(out, label) loss.backward() trainer.step(data.shape[0]) metric.update(label, out) for name, value in metric.get(): logging.info('Epoch[%04d] Train-%s=%f ...', epoch, name, value) _result = None map(lambda x: x.reset(), [testI, metric]) for data, label in testI: out = net.forward(data) self.forDebug(out) metric.update(label, out) for name, value in metric.get(): if name == monitor: _result = value logging.info('Epoch[%04d] Validation-%s=%f', epoch, name, value) if _result > result: result = _result name = '%s/%04d-%3.3f%%.params' % (model, epoch, result * 100) net.save_params(name) logging.info('Save params to %s ...', name) logging.info('Epoch[%04d] done ...\n' % epoch)
from mxnet import autograd from mxnet.gluon import trainer import time import os os.environ['MXNET_CUDNN_AUTOTUNE_DEFAULT'] = '0' vgg16 = model_zoo.vision.vgg16(ctx=mx.gpu()) vgg16.initialize(ctx=mx.gpu()) criterion = loss.SoftmaxCrossEntropyLoss() update = trainer.Trainer(vgg16.collect_params(), optimizer='sgd') begin = time.time() bs = 60 for i in range(1000): print(i) inputs = mx.nd.normal(shape=(bs, 3, 224, 224), ctx=mx.gpu()) labels = mx.nd.array([0] * bs, ctx=mx.gpu()) with autograd.record(): logits = vgg16(inputs) loss = criterion(logits, labels) loss.backward() update.step(batch_size=bs) print("time ", time.time() - begin) # bs=10, iter=1000, time 10.77, Menory=3441M # upper bound=60
def train(self): model = self.getModel() if not os.path.exists(model): os.mkdir(model) ctx = mx.cpu(self.gpu) net, myLoss = self.getNet(ctx) trainI, testI = self.getData(ctx) metric, monitor = self.getMetric() trainer = self.getTrainer( net.collect_params(), trainI.iters) logging.info('') result, epochs = 0, self.getEpoch() for epoch in range(1, epochs+1): train_l_sum = mx.nd.array([0], ctx=ctx) logging.info('Epoch[%04d] start ...' % epoch) list(map(lambda x: x.reset(), [trainI, metric])) for batch_i, (data, label) in enumerate(trainI): with autograd.record(): out = net.forward(data)#2048,590 self.forDebug(out) loss = myLoss(out, label) loss.backward() # grads = [p.grad(ctx) for p in net.collect_params().values()] # mx.gluon.utils.clip_global_norm( # grads, .2 * 5 * data.shape[0]) trainer.step(data.shape[0])#trainer.step(batch_size) print('train loss:', loss.mean().asnumpy()) ############################################### # train_l_sum += loss.sum() / data.shape[0] # eval_period = 1 # if batch_i % eval_period == 0 and batch_i > 0: # cur_l = train_l_sum / eval_period # print('epoch %d, batch %d, train loss %.6f, perplexity %.2f' # % (epoch, batch_i, cur_l.asscalar(), # cur_l.exp().asscalar())) # train_l_sum = mx.nd.array([0], ctx=ctx) ############################################# metric.update(label, out) for name, value in metric.get(): logging.info('Epoch[%04d] Train-%s=%f ...', epoch, name, value) _result = None list(map(lambda x: x.reset(), [testI, metric])) l_sum = mx.nd.array([0], ctx=ctx) n = 0 for data, label in testI: out = net.forward(data) l = myLoss(out, label) l_sum += l.sum() n += l.size self.forDebug(out) metric.update(label, out) print('valid loss:', (l_sum / n).mean().asnumpy()) for name, value in metric.get(): if name == monitor : _result = value logging.info('Epoch[%04d] Validation-%s=%f', epoch, name, value) if _result > result: result = _result name = '%s/%04d-%3.3f%%.params' % (model, epoch, result*100) net.save_params(name) logging.info('Save params to %s ...', name) logging.info('Epoch[%04d] done ...\n' % epoch)