def check_quantize_model(qdtype): def check_params(params, qparams, qsym=None): if qsym is None: assert len(params) == len(qparams) for k, v in params.items(): assert k in qparams assert same(v.asnumpy(), qparams[k].asnumpy()) else: qparams_ground_truth = mx.contrib.quant._quantize_params(qsym, params) assert len(qparams) == len(qparams_ground_truth) for k, v in qparams_ground_truth.items(): assert k in qparams assert same(v.asnumpy(), qparams[k].asnumpy()) def check_qsym_calibrated(qsym): attrs = qsym.attr_dict() for k, v in attrs.items(): if k.find('requantize_') != -1: assert 'min_calib_range' in v assert 'max_calib_range' in v def check_qsym_qdtype(qsym, qdtype): attrs = qsym.attr_dict() for k, v in attrs.items(): if k.find('_quantize') != -1: assert 'out_type' in v assert v['out_type'] == qdtype sym = get_fp32_sym() mod = Module(symbol=sym) batch_size = 4 data_shape = (batch_size, 4, 10, 10) label_shape = (batch_size, 10) mod.bind(data_shapes=[('data', data_shape)], label_shapes=[('softmax_label', label_shape)]) mod.init_params() arg_params, aux_params = mod.get_params() qsym, qarg_params, qaux_params = mx.contrib.quant.quantize_model(sym=sym, arg_params=arg_params, aux_params=aux_params, ctx=mx.current_context(), quantized_dtype=qdtype, calib_mode='none') check_params(arg_params, qarg_params, qsym) check_params(aux_params, qaux_params) calib_data = mx.nd.random.uniform(shape=data_shape) calib_data = NDArrayIter(data=calib_data) calib_data = DummyIter(calib_data) qsym, qarg_params, qaux_params = mx.contrib.quant.quantize_model(sym=sym, arg_params=arg_params, aux_params=aux_params, ctx=mx.current_context(), quantized_dtype=qdtype, calib_mode='naive', calib_data=calib_data, num_calib_examples=20) check_params(arg_params, qarg_params, qsym) check_params(aux_params, qaux_params) check_qsym_calibrated(qsym) check_qsym_qdtype(qsym, qdtype)
def check_quantize_whole_model(out_type): batch_size = 4 data_shape = (batch_size, 4, 10, 10) data = mx.sym.Variable('data') conv0 = mx.sym.Convolution(data, kernel=(1, 1), num_filter=16, name='conv0') sym = mx.sym.Convolution(conv0, kernel=(1, 1), num_filter=16, name='conv1') sym_sg = sym.get_backend_symbol('MKLDNN_QUANTIZE') mod = Module(symbol=sym, label_names=None) mod.bind(for_training=False, data_shapes=[('data', data_shape)]) mod.init_params(mx.init.Normal(0.5)) arg_params, aux_params = mod.get_params() excluded_sym_names = [] calib_data = mx.nd.random.uniform(shape=data_shape) calib_data = mx.io.NDArrayIter(data=calib_data) calib_data = DummyIter(calib_data) calib_layer = lambda name: name.endswith('_output') qsym, qarg_params, qaux_params = mx.contrib.quant.quantize_model(sym=sym_sg, arg_params=arg_params, aux_params=aux_params, ctx=mx.current_context(), excluded_sym_names=excluded_sym_names, quantized_dtype=out_type, calib_mode='naive', calib_data=calib_data, calib_layer=calib_layer, label_names=None, num_calib_examples=1) qsym = qsym.get_backend_symbol('MKLDNN_QUANTIZE') check_qsym_forward(qsym, qarg_params, qaux_params, data_shape)
def check_quantize(sym, data_shape, out_type, name='conv', check_calibration=True, gluon_forward=False): sg_pass_name = config[name][SG_PASS_NAME] post_sg_pass_name = config[name][POST_SG_PASS_NAME] fc = mx.sym.FullyConnected(data=sym, num_hidden=10, flatten=True, name='fc_softmax') if gluon_forward == True: sym = fc sym_sg = sym.get_backend_symbol(sg_pass_name) mod = Module(symbol=sym, label_names=[]) mod.bind(for_training=False, data_shapes=[('data', data_shape)]) else: sym = mx.sym.SoftmaxOutput(data=fc, name='softmax') sym_sg = sym.get_backend_symbol(sg_pass_name) label_shape = (data_shape[0], 10) mod = Module(symbol=sym) mod.bind(for_training=False, data_shapes=[('data', data_shape)], label_shapes=[('softmax_label', label_shape)]) mod.init_params(mx.init.Normal(0.5)) arg_params, aux_params = mod.get_params() data = [mx.random.uniform(-1, 1, shape=shape, ctx=mx.current_context()) for _, shape in mod.data_shapes] batch = mx.io.DataBatch(data, []) mod.forward(batch, is_train=False) for output in mod.get_outputs(): output.wait_to_read() ref_out = mod.get_outputs() excluded_sym_names = [] if mx.current_context() == mx.cpu() and gluon_forward == True: excluded_sym_names += ['sg_mkldnn_fully_connected_0'] excluded_sym_names += ['fc_softmax'] calib_data = mx.nd.random.uniform(shape=data_shape) calib_data = NDArrayIter(data=calib_data) calib_data = DummyIter(calib_data) calib_layer = lambda name: name.endswith('_output') qsym, qarg_params, qaux_params = mx.contrib.quant.quantize_model(sym=sym_sg, arg_params=arg_params, aux_params=aux_params, ctx=mx.current_context(), excluded_sym_names=excluded_sym_names, quantized_dtype=out_type, calib_mode='naive', calib_data=calib_data, calib_layer=calib_layer, num_calib_examples=5) qsym = qsym.get_backend_symbol(post_sg_pass_name) if check_calibration: check_qsym_calibrated(qsym, out_type, name=name) if gluon_forward == True: check_qsym_gluon_forward(qsym, qarg_params, qaux_params, data_shape) else: check_qsym_dummy_forward(qsym, batch, data_shape, label_shape) quantized_out = check_qsym_forward(qsym, qarg_params, qaux_params, batch, data_shape, label_shape) for i in range(len(ref_out)): assert_almost_equal(ref_out[i].asnumpy(), quantized_out[i].asnumpy(), atol = 1)
def check_quantize(sym, data_shape, out_type, name='conv', check_calibration=True, gluon_forward=False, check_scale_align=False): if name in config: name = config[name][OP_NAME] sym_sg = sym.get_backend_symbol(QUANTIZE_SG_PASS_NAME) mod = Module(symbol=sym, label_names=None) mod.bind(for_training=False, data_shapes=[('data', data_shape)]) mod.init_params(mx.init.Normal(0.5)) arg_params, aux_params = mod.get_params() if out_type == 'uint8': data = [mx.random.uniform(0.0, 1.0, shape=shape, ctx=mx.current_context()) for _, shape in mod.data_shapes] else: data = [mx.random.uniform(-1.0, 1.0, shape=shape, ctx=mx.current_context()) for _, shape in mod.data_shapes] batch = mx.io.DataBatch(data, []) mod.forward(batch, is_train=False) for output in mod.get_outputs(): output.wait_to_read() ref_out = mod.get_outputs() excluded_sym_names = [] excluded_op_names = [] if mx.current_context() == mx.cpu() and gluon_forward == True: excluded_op_names += ['_sg_mkldnn_fully_connected'] calib_data = CalibIter(batch, data_shape, 1) qsym, qarg_params, qaux_params = mx.contrib.quant.quantize_model(sym=sym_sg, arg_params=arg_params, aux_params=aux_params, ctx=mx.current_context(), excluded_sym_names=excluded_sym_names, excluded_op_names=excluded_op_names, quantized_dtype=out_type, calib_mode='naive', calib_data=calib_data, calib_layer=None, label_names=None, num_calib_examples=1) qsym = qsym.get_backend_symbol(QUANTIZE_SG_PASS_NAME) if check_calibration: check_qsym_calibrated(qsym, out_type, name=name) if check_scale_align: check_qsym_scale_align(qsym) if gluon_forward == True: check_qsym_gluon_forward(qsym, qarg_params, qaux_params, data_shape) else: quantized_out = check_qsym_forward(qsym, qarg_params, qaux_params, batch, data_shape) for i in range(len(ref_out)): min_range = mx.nd.min(ref_out[i]).asscalar() max_range = mx.nd.max(ref_out[i]).asscalar() atol = 0.1 * max(abs(min_range), abs(max_range)) assert_almost_equal_with_err(quantized_out[i].asnumpy(), ref_out[i].asnumpy(), rtol=0.1, atol=atol, etol=0.2) check_qsym_dummy_forward(qsym, batch, data_shape)
def check_quantize(sym, data_shape, check_conv=True): fc = mx.sym.FullyConnected(data=sym, num_hidden=10, flatten=True, name='fc') sym = mx.sym.SoftmaxOutput(data=fc, name='softmax') sym_sg = sym.get_backend_symbol("MKLDNN") label_shape = (data_shape[0], 10) mod = Module(symbol=sym) mod.bind(for_training=False, data_shapes=[('data', data_shape)], label_shapes=[('softmax_label', label_shape)]) mod.init_params(mx.init.Normal(0.5)) arg_params, aux_params = mod.get_params() data = [ mx.random.uniform(-1, 1, shape=shape, ctx=mx.current_context()) for _, shape in mod.data_shapes ] batch = mx.io.DataBatch(data, []) mod.forward(batch, is_train=False) for output in mod.get_outputs(): output.wait_to_read() ref_out = mod.get_outputs() excluded_sym_names = [] if mx.current_context() == mx.cpu(): excluded_sym_names += ['fc'] calib_data = mx.nd.random.uniform(shape=data_shape) calib_data = NDArrayIter(data=calib_data) calib_data = DummyIter(calib_data) calib_layer = lambda name: name.endswith('_output') qsym, qarg_params, qaux_params = mx.contrib.quant.quantize_model( sym=sym_sg, arg_params=arg_params, aux_params=aux_params, ctx=mx.current_context(), excluded_sym_names=excluded_sym_names, quantized_dtype='uint8', calib_mode='naive', calib_data=calib_data, calib_layer=calib_layer, calib_quantize_op=True, num_calib_examples=5) qsym = qsym.get_backend_symbol("MKLDNN_POST_QUANTIZE") if check_conv: check_qsym_calibrated(qsym) quantized_out = check_qsym_forward(qsym, qarg_params, qaux_params, batch, data_shape, label_shape) for i in range(len(ref_out)): assert_almost_equal(ref_out[i].asnumpy(), quantized_out[i].asnumpy(), atol=1) check_qsym_dummy_forward(qsym, batch, data_shape, label_shape)
def check_quantize(sym, data_shape, check_conv=True): fc = mx.sym.FullyConnected(data=sym, num_hidden=10, flatten=True, name='fc') sym = mx.sym.SoftmaxOutput(data=fc, name='softmax') sym_sg = sym.get_backend_symbol("MKLDNN") label_shape = (data_shape[0], 10) mod = Module(symbol=sym) mod.bind(for_training=False, data_shapes=[('data', data_shape)], label_shapes=[('softmax_label', label_shape)]) mod.init_params(mx.init.Normal(0.5)) arg_params, aux_params = mod.get_params() data = [mx.random.uniform(-1, 1, shape=shape, ctx=mx.current_context()) for _, shape in mod.data_shapes] batch = mx.io.DataBatch(data, []) mod.forward(batch, is_train=False) for output in mod.get_outputs(): output.wait_to_read() ref_out = mod.get_outputs() excluded_sym_names = [] if mx.current_context() == mx.cpu(): excluded_sym_names += ['fc'] calib_data = mx.nd.random.uniform(shape=data_shape) calib_data = NDArrayIter(data=calib_data) calib_data = DummyIter(calib_data) calib_layer = lambda name: name.endswith('_output') qsym, qarg_params, qaux_params = mx.contrib.quant.quantize_model(sym=sym_sg, arg_params=arg_params, aux_params=aux_params, ctx=mx.current_context(), excluded_sym_names=excluded_sym_names, quantized_dtype='uint8', calib_mode='naive', calib_data=calib_data, calib_layer=calib_layer, calib_quantize_op=True, num_calib_examples=5) qsym = qsym.get_backend_symbol("MKLDNN_POST_QUANTIZE") if check_conv: check_qsym_calibrated(qsym) quantized_out = check_qsym_forward(qsym, qarg_params, qaux_params, batch, data_shape, label_shape) for i in range(len(ref_out)): assert_almost_equal(ref_out[i].asnumpy(), quantized_out[i].asnumpy(), atol = 1) check_qsym_dummy_forward(qsym, batch, data_shape, label_shape)
class Solver(object): def __init__(self, symbol, data_names, label_names, data_shapes, label_shapes, logger=logging, context=mx.cpu(), work_load_list=None, fixed_param_names=None): self.symbol = symbol self.data_names = data_names self.label_names = label_names self.data_shapes = data_shapes self.label_shapes = label_shapes self.context = context self.work_load_list = work_load_list self.fixed_param_names = fixed_param_names if logger is None: logger = logging.getLogger() logger.setLevel(logging.INFO) self.logger = logger self.module = Module(symbol=self.symbol, data_names=self.data_names, label_names=self.label_names, logger=self.logger, context=self.context, work_load_list=self.work_load_list, fixed_param_names=self.fixed_param_names) def fit(self, train_data, eval_data=None, eval_metric='acc', validate_metric=None, work_load_list=None, epoch_end_callback=None, batch_end_callback=None, fixed_param_prefix=None, initializer=None, arg_params=None, aux_params=None, allow_missing=False, optimizer=None, optimizer_params=None, begin_epoch=0, num_epoch=None, kvstore='device', teacher_modules=None): if type(teacher_modules) is not list: teacher_modules = [teacher_modules] self.module.bind(data_shapes=self.data_shapes, label_shapes=self.label_shapes, for_training=True) self.module.init_params(initializer=initializer, arg_params=arg_params, aux_params=aux_params, allow_missing=allow_missing) self.module.init_optimizer(kvstore=kvstore, optimizer=optimizer, optimizer_params=optimizer_params) if validate_metric is None: validate_metric = eval_metric if not isinstance(eval_metric, metric.EvalMetric): eval_metric = metric.create(eval_metric) # training loop for epoch in range(begin_epoch, num_epoch): tic = time.time() eval_metric.reset() nbatch = 0 data_iter = iter(train_data) end_of_batch = False next_data_batch = next(data_iter) while not end_of_batch: data_batch = next_data_batch if teacher_modules[0] is not None: for teacher_module in teacher_modules: teacher_module.forward(data_batch=data_batch, is_train=True) transfer_label = teacher_module.get_outputs() data_batch.label = data_batch.label + transfer_label self.module.forward(data_batch, is_train=True) self.module.backward() self.module.update() try: next_data_batch = next(data_iter) except StopIteration: end_of_batch = True self.module.update_metric(eval_metric, data_batch.label) if batch_end_callback is not None: batch_end_params = BatchEndParam(epoch=epoch, nbatch=nbatch, eval_metric=eval_metric, locals=locals()) for callback in _as_list(batch_end_callback): callback(batch_end_params) nbatch += 1 for name, val in eval_metric.get_name_value(): self.logger.info('Epoch[%d] Train-%s=%f', epoch, name, val) toc = time.time() self.logger.info('Epoch[%d] Time cost=%.3f', epoch, (toc - tic)) arg_params, aux_params = self.module.get_params() self.module.set_params(arg_params, aux_params) if epoch_end_callback is not None: for callback in _as_list(epoch_end_callback): callback(epoch, self.symbol, arg_params, aux_params) if eval_data: res = self.module.score(eval_data, validate_metric, score_end_callback=None, batch_end_callback=None, reset=True, epoch=epoch) for name, val in res: self.logger.info('Epoch[%d] Validation-%s=%f', epoch, name, val) train_data.reset()
def check_quantize_model(qdtype): if is_test_for_native_cpu(): print( 'skipped testing test_quantize_model_with_forward for native cpu since it is not supported yet' ) return elif qdtype == 'int8' and is_test_for_mkldnn(): print( 'skipped testing test_quantize_model_with_forward for mkldnn cpu int8 since it is not supported yet' ) return elif qdtype == 'uint8' and is_test_for_gpu(): print( 'skipped testing test_quantize_model_with_forward for gpu uint8 since it is not supported yet' ) return def check_params(params, qparams, qsym=None): if qsym is None: assert len(params) == len(qparams) for k, v in params.items(): assert k in qparams assert same(v.asnumpy(), qparams[k].asnumpy()) else: qparams_ground_truth = mx.contrib.quant._quantize_params( qsym, params, th_dict={}) assert len(qparams) == len(qparams_ground_truth) for k, v in qparams_ground_truth.items(): assert k in qparams assert same(v.asnumpy(), qparams[k].asnumpy()) def check_qsym_calibrated(qsym): attrs = qsym.attr_dict() for k, v in attrs.items(): if k.find('requantize_') != -1: assert 'min_calib_range' in v assert 'max_calib_range' in v def check_qsym_qdtype(qsym, qdtype): attrs = qsym.attr_dict() for k, v in attrs.items(): if k.find('_quantize') != -1: assert 'out_type' in v assert v['out_type'] == qdtype def check_qsym_forward(qsym, qarg_params, qaux_params, data_shape, label_shape): mod = mx.mod.Module(symbol=qsym, context=mx.current_context()) mod.bind(for_training=False, data_shapes=[('data', data_shape)], label_shapes=[('softmax_label', label_shape)]) mod.set_params(qarg_params, qaux_params) data = [ mx.random.uniform(-1.0, 1.0, shape=shape) for _, shape in mod.data_shapes ] batch = mx.io.DataBatch(data, []) mod.forward(batch, is_train=False) for output in mod.get_outputs(): output.wait_to_read() sym = get_fp32_residual() batch_size = 4 data_shape = (batch_size, 4, 10, 10) label_shape = (batch_size, 10) length = batch_size # specify num of outputs from split op msym = get_fp32_sym_with_multiple_outputs(length) msym_label_shape = (length, 10) msym_data_shape = (length, 4, 4, 10, 10) for s, dshape, lshape in zip((sym, msym), (data_shape, msym_data_shape), (label_shape, msym_label_shape)): mod = Module(symbol=s) mod.bind(data_shapes=[('data', dshape)], label_shapes=[('softmax_label', lshape)]) mod.init_params() arg_params, aux_params = mod.get_params() excluded_names = [] if mx.current_context() == mx.cpu(): excluded_names += ['fc'] excluded_names += ['concat'] optional_names = ['pool0'] for skip_optional_names in [False, True]: exclude_sym_names = [] if skip_optional_names: excluded_sym_names = excluded_names else: excluded_sym_names = excluded_names + optional_names qsym, qarg_params, qaux_params = mx.contrib.quant.quantize_model( sym=s, arg_params=arg_params, aux_params=aux_params, excluded_sym_names=excluded_sym_names, ctx=mx.current_context(), quantized_dtype=qdtype, calib_mode='none') check_params(arg_params, qarg_params, qsym) check_params(aux_params, qaux_params) check_qsym_forward(qsym, qarg_params, qaux_params, dshape, lshape) calib_data = mx.nd.random.uniform(shape=dshape) calib_data = NDArrayIter(data=calib_data, batch_size=batch_size) calib_data = DummyIter(calib_data) qsym, qarg_params, qaux_params = mx.contrib.quant.quantize_model( sym=s, arg_params=arg_params, aux_params=aux_params, excluded_sym_names=excluded_sym_names, ctx=mx.current_context(), quantized_dtype=qdtype, calib_mode='naive', calib_data=calib_data, num_calib_examples=20) check_params(arg_params, qarg_params, qsym) check_params(aux_params, qaux_params) check_qsym_calibrated(qsym) check_qsym_qdtype(qsym, qdtype) check_qsym_forward(qsym, qarg_params, qaux_params, dshape, lshape)
def check_quantize_model(qdtype): if is_test_for_native_cpu(): print( 'skipped testing test_quantize_model_with_forward for native cpu since it is not supported yet' ) return elif qdtype == 'int8' and is_test_for_mkldnn(): print( 'skipped testing test_quantize_model_with_forward for mkldnn cpu int8 since it is not supported yet' ) return elif qdtype == 'uint8' and is_test_for_gpu(): print( 'skipped testing test_quantize_model_with_forward for gpu uint8 since it is not supported yet' ) return def check_params(params, qparams, qsym=None): if qsym is None: assert len(params) == len(qparams) for k, v in params.items(): assert k in qparams assert same(v.asnumpy(), qparams[k].asnumpy()) else: qparams_ground_truth = mx.contrib.quant._quantize_params( qsym, params, th_dict={}) assert len(qparams) == len(qparams_ground_truth) for k, v in qparams_ground_truth.items(): assert k in qparams assert same(v.asnumpy(), qparams[k].asnumpy()) def check_qsym_calibrated(qsym): attrs = qsym.attr_dict() for k, v in attrs.items(): if k.find('requantize_') != -1: assert 'min_calib_range' in v assert 'max_calib_range' in v def check_qsym_qdtype(qsym, qdtype): attrs = qsym.attr_dict() for k, v in attrs.items(): if k.find('_quantize') != -1: assert 'out_type' in v assert v['out_type'] == qdtype def check_qsym_forward(qsym, qarg_params, qaux_params, data_shape): mod = mx.mod.Module(symbol=qsym, label_names=None, context=mx.current_context()) mod.bind(for_training=False, data_shapes=[('data', data_shape)]) mod.set_params(qarg_params, qaux_params) data = [ mx.random.uniform(-1.0, 1.0, shape=shape) for _, shape in mod.data_shapes ] batch = mx.io.DataBatch(data, []) mod.forward(batch, is_train=False) for output in mod.get_outputs(): output.wait_to_read() batch_size = 4 dshape = (batch_size, 4, 10, 10) data = mx.sym.Variable('data') sym = mx.sym.Convolution(data, kernel=(1, 1), num_filter=16, name='conv0') mod = Module(symbol=sym, label_names=None) mod.bind(data_shapes=[('data', dshape)]) mod.init_params() arg_params, aux_params = mod.get_params() excluded_sym_names = [] qsym, qarg_params, qaux_params = mx.contrib.quant.quantize_model( sym=sym, arg_params=arg_params, aux_params=aux_params, excluded_sym_names=excluded_sym_names, ctx=mx.current_context(), quantized_dtype=qdtype, calib_mode='none') check_params(arg_params, qarg_params, qsym) check_params(aux_params, qaux_params) check_qsym_forward(qsym, qarg_params, qaux_params, dshape) calib_data = mx.nd.random.uniform(shape=dshape) calib_data = NDArrayIter(data=calib_data, batch_size=batch_size) calib_data = DummyIter(calib_data) qsym, qarg_params, qaux_params = mx.contrib.quant.quantize_model( sym=sym, arg_params=arg_params, aux_params=aux_params, excluded_sym_names=excluded_sym_names, ctx=mx.current_context(), quantized_dtype=qdtype, calib_mode='naive', calib_data=calib_data, num_calib_examples=20) check_params(arg_params, qarg_params, qsym) check_params(aux_params, qaux_params) check_qsym_calibrated(qsym) check_qsym_qdtype(qsym, qdtype) check_qsym_forward(qsym, qarg_params, qaux_params, dshape)
def check_quantize_model(qdtype): if is_test_for_native_cpu(): print( 'skipped testing quantize_model for native cpu since it is not supported yet' ) return elif qdtype == 'int8' and is_test_for_mkldnn(): print( 'skipped testing quantize_model for mkldnn cpu int8 since it is not supported yet' ) return elif qdtype == 'uint8' and is_test_for_gpu(): print( 'skipped testing quantize_model for gpu uint8 since it is not supported yet' ) return def check_params(params, qparams, qsym=None): if qsym is None: assert len(params) == len(qparams) for k, v in params.items(): assert k in qparams assert same(v.asnumpy(), qparams[k].asnumpy()) else: qparams_ground_truth = mx.contrib.quant._quantize_params( qsym, params, th_dict={}) assert len(qparams) == len(qparams_ground_truth) for k, v in qparams_ground_truth.items(): assert k in qparams assert same(v.asnumpy(), qparams[k].asnumpy()) def check_qsym_calibrated(qsym): attrs = qsym.attr_dict() for k, v in attrs.items(): if k.find('requantize_') != -1: assert 'min_calib_range' in v assert 'max_calib_range' in v def check_qsym_qdtype(qsym, qdtype): attrs = qsym.attr_dict() for k, v in attrs.items(): if k.find('_quantize') != -1: assert 'out_type' in v assert v['out_type'] == qdtype sym = get_fp32_sym() batch_size = 4 label_shape = (batch_size, 10) data_shape = (batch_size, 4, 10, 10) length = batch_size # specify num of outputs from split op msym = get_fp32_sym_with_multiple_outputs(length) msym_label_shape = (length, 10) msym_data_shape = (length, 4, 4, 10, 10) for s, dshape, lshape in zip((sym, msym), (data_shape, msym_data_shape), (label_shape, msym_label_shape)): mod = Module(symbol=s) mod.bind(data_shapes=[('data', dshape)], label_shapes=[('softmax_label', lshape)]) mod.init_params() arg_params, aux_params = mod.get_params() qsym, qarg_params, qaux_params = mx.contrib.quant.quantize_model( sym=s, arg_params=arg_params, aux_params=aux_params, ctx=mx.current_context(), quantized_dtype=qdtype, calib_mode='none') check_params(arg_params, qarg_params, qsym) check_params(aux_params, qaux_params) calib_data = mx.nd.random.uniform(shape=dshape) calib_data = NDArrayIter(data=calib_data, batch_size=batch_size) calib_data = DummyIter(calib_data) qsym, qarg_params, qaux_params = mx.contrib.quant.quantize_model( sym=s, arg_params=arg_params, aux_params=aux_params, ctx=mx.current_context(), quantized_dtype=qdtype, calib_mode='naive', calib_data=calib_data, num_calib_examples=20) check_params(arg_params, qarg_params, qsym) check_params(aux_params, qaux_params) check_qsym_calibrated(qsym) check_qsym_qdtype(qsym, qdtype)
def check_quantize_model(qdtype): if is_test_for_native_cpu(): print('skipped testing quantized_residual_unit for native cpu since it is not supported yet') return elif qdtype == 'int8' and is_test_for_mkldnn(): print('skipped testing quantized_residual_unit for mkldnn cpu int8 since it is not supported yet') return elif qdtype == 'uint8' and is_test_for_gpu(): print('skipped testing quantized_residual_unit for gpu uint8 since it is not supported yet') return def check_params(params, qparams, qsym=None): if qsym is None: assert len(params) == len(qparams) for k, v in params.items(): assert k in qparams assert same(v.asnumpy(), qparams[k].asnumpy()) else: qparams_ground_truth = mx.contrib.quant._quantize_params(qsym, params) assert len(qparams) == len(qparams_ground_truth) for k, v in qparams_ground_truth.items(): assert k in qparams assert same(v.asnumpy(), qparams[k].asnumpy()) def check_qsym_calibrated(qsym): attrs = qsym.attr_dict() for k, v in attrs.items(): if k.find('requantize_') != -1: assert 'min_calib_range' in v assert 'max_calib_range' in v def check_qsym_qdtype(qsym, qdtype): attrs = qsym.attr_dict() for k, v in attrs.items(): if k.find('_quantize') != -1: assert 'out_type' in v assert v['out_type'] == qdtype def check_qsym_forward(qsym, qarg_params, qaux_params, data_shape, label_shape): mod = mx.mod.Module(symbol=qsym, context=mx.current_context()) mod.bind(for_training=False, data_shapes=[('data', data_shape)], label_shapes=[('softmax_label', label_shape)]) mod.set_params(qarg_params, qaux_params) data = [mx.random.uniform(-1.0, 1.0, shape=shape) for _, shape in mod.data_shapes] batch = mx.io.DataBatch(data, []) mod.forward(batch, is_train=False) for output in mod.get_outputs(): output.wait_to_read() sym = get_fp32_residual() mod = Module(symbol=sym) batch_size = 4 data_shape = (batch_size, 4, 10, 10) label_shape = (batch_size, 10) mod.bind(data_shapes=[('data', data_shape)], label_shapes=[('softmax_label', label_shape)]) mod.init_params() arg_params, aux_params = mod.get_params() excluded_sym_names = [] if mx.current_context() == mx.cpu(): excluded_sym_names += ['fc'] qsym, qarg_params, qaux_params = mx.contrib.quant.quantize_model(sym=sym, arg_params=arg_params, aux_params=aux_params, excluded_sym_names=excluded_sym_names, ctx=mx.current_context(), quantized_dtype=qdtype, calib_mode='none') check_params(arg_params, qarg_params, qsym) check_params(aux_params, qaux_params) check_qsym_forward(qsym, qarg_params, qaux_params, data_shape, label_shape) calib_data = mx.nd.random.uniform(shape=data_shape) calib_data = NDArrayIter(data=calib_data) calib_data = DummyIter(calib_data) qsym, qarg_params, qaux_params = mx.contrib.quant.quantize_model(sym=sym, arg_params=arg_params, aux_params=aux_params, excluded_sym_names=excluded_sym_names, ctx=mx.current_context(), quantized_dtype=qdtype, calib_mode='naive', calib_data=calib_data, num_calib_examples=20) check_params(arg_params, qarg_params, qsym) check_params(aux_params, qaux_params) check_qsym_calibrated(qsym) check_qsym_qdtype(qsym, qdtype) check_qsym_forward(qsym, qarg_params, qaux_params, data_shape, label_shape)
class Solver(object): def __init__( self, symbol, data_names, label_names, data_shapes, label_shapes, logger=logging, context=mx.cpu(), work_load_list=None, fixed_param_names=None, allow_missing=False, # for evaluate fold bn to create eval symbol config=None): self.symbol = symbol self.data_names = data_names self.label_names = label_names self.data_shapes = data_shapes self.label_shapes = label_shapes self.context = context self.work_load_list = work_load_list self.fixed_param_names = fixed_param_names if logger is None: logger = logging.getLogger() logger.setLevel(logging.INFO) self.logger = logger self.module = Module(symbol=self.symbol, data_names=self.data_names, label_names=self.label_names, logger=self.logger, context=self.context, work_load_list=self.work_load_list, fixed_param_names=self.fixed_param_names) # for fold bn self.config = config def fit(self, train_data, eval_data=None, eval_metric='acc', validate_metric=None, work_load_list=None, epoch_end_callback=None, batch_end_callback=None, fixed_param_prefix=None, initializer=None, arg_params=None, aux_params=None, allow_missing=False, optimizer=None, optimizer_params=None, begin_epoch=0, num_epoch=None, kvstore='device'): self.module.bind(data_shapes=self.data_shapes, label_shapes=self.label_shapes, for_training=True) self.module.init_params(initializer=initializer, arg_params=arg_params, aux_params=aux_params, allow_missing=allow_missing) self.module.init_optimizer(kvstore=kvstore, optimizer=optimizer, optimizer_params=optimizer_params) if validate_metric is None: validate_metric = eval_metric if not isinstance(eval_metric, metric.EvalMetric): eval_metric = metric.create(eval_metric) temp_count = 0 # # test model size by saving params of model # arg_params, aux_params = self.module.get_params() # for callback in _as_list(epoch_end_callback): # callback(0, self.symbol, arg_params, aux_params) # raise NotImplementedError # training loop for epoch in range(begin_epoch, num_epoch): train_time = AverageMeter() kvstore_sync_time = AverageMeter() get_data_time = AverageMeter() iter_total_time = AverageMeter() tic = time.time() eval_metric.reset() nbatch = 0 data_iter = iter(train_data) end_of_batch = False next_data_batch = next(data_iter) while not end_of_batch: start_time = time.time() data_batch = next_data_batch self.module.forward(data_batch, is_train=True) self.module.backward() # ndarray.waitall() train_time.update(time.time() - start_time) self.module.update() # ndarray.waitall() kvstore_sync_time.update(time.time() - start_time) try: next_data_batch = next(data_iter) except StopIteration: end_of_batch = True # ndarray.waitall() get_data_time.update(time.time() - start_time) if isinstance(data_batch, list): self.module.update_metric(eval_metric, [db.label for db in data_batch], pre_sliced=True) else: self.module.update_metric(eval_metric, data_batch.label) # ndarray.waitall() iter_total_time.update(time.time() - start_time) if batch_end_callback is not None: # batch_end_params = BatchEndParam(epoch=epoch, nbatch=nbatch, # eval_metric=eval_metric, # locals=locals()) batch_end_params = BatchEndParam( epoch=epoch, nbatch=nbatch, eval_metric=eval_metric, locals=locals(), rank=kvstore.rank, total_iter=temp_count, cur_data_time=get_data_time.val, avg_data_time=get_data_time.avg, cur_batch_time=train_time.val, avg_batch_time=train_time.avg, cur_kvstore_sync_time=kvstore_sync_time.val, avg_kvstore_sync_time=kvstore_sync_time.avg, cur_iter_total_time=iter_total_time.val, avg_iter_total_time=iter_total_time.avg) for callback in _as_list(batch_end_callback): callback(batch_end_params) nbatch += 1 temp_count += 1 for name, val in eval_metric.get_name_value(): self.logger.info('Epoch[%d] Train-%s=%f', epoch, name, val) toc = time.time() self.logger.info('Epoch[%d] Time cost=%.3f', epoch, (toc - tic)) arg_params, aux_params = self.module.get_params() self.module.set_params(arg_params, aux_params) if epoch_end_callback is not None and kvstore.rank == 0: for callback in _as_list(epoch_end_callback): callback(epoch, self.symbol, arg_params, aux_params) if eval_data: if self.config.network == 'mobilenet_int8_foldbn': # for fold bn to create inference symbol total_params_path = "./model/%s-%04d.params" % ( self.config.model_prefix, epoch + 1) # total_params_path = "./model/mobilenet_flodbn_0904/mobilenet_int8_flodbn_imagenet_retrain_80_pertensor-fold-0100.params" # _, arg_params, aux_params = mx.model.load_checkpoint('./model/mobilenet_flodbn_0904/mobilenet_int8_flodbn_imagenet_retrain_80_pertensor-fold', 100) import os assert os.path.exists( total_params_path ), "please provide the correct total_params_path for foldbn eval" eval_sym = eval(self.config.network)( num_classes=self.config.num_classes, quant_mod=self.config.quant_mod, delay_quant=self.config.delay_quant, is_weight_perchannel=self.config.is_weight_perchannel, total_params_path=total_params_path, quantize_flag=self.config.quantize_flag) eval_module = Module( symbol=eval_sym, data_names=self.data_names, label_names=self.label_names, logger=self.logger, context=self.context, work_load_list=self.work_load_list, fixed_param_names=self.fixed_param_names) eval_module.bind(data_shapes=self.data_shapes, label_shapes=self.label_shapes, for_training=False) eval_module.init_params(initializer=initializer, arg_params=arg_params, aux_params=aux_params) res = eval_module.score(eval_data, validate_metric, score_end_callback=None, batch_end_callback=None, reset=True, epoch=epoch) else: res = self.module.score(eval_data, validate_metric, score_end_callback=None, batch_end_callback=None, reset=True, epoch=epoch) for name, val in res: self.logger.info('Epoch[%d] Validation-%s=%f', epoch, name, val) train_data.reset()