def linear_fc_quant(self, activation_quant_type, weight_quantize_type, for_ci=True): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): loss = linear_fc(3) opt = fluid.optimizer.Adam(learning_rate=0.001) opt.minimize(loss) place = fluid.CPUPlace() graph = IrGraph(core.Graph(main.desc), for_test=False) transform_pass = QuantizationTransformPass( scope=fluid.global_scope(), place=place, activation_quantize_type=activation_quant_type, weight_quantize_type=weight_quantize_type) transform_pass.apply(graph) if not for_ci: marked_nodes = set() for op in graph.all_op_nodes(): if op.name().find('quantize') > -1: marked_nodes.add(op) graph.draw('.', 'quantize_fc_' + activation_quant_type, marked_nodes) program = graph.to_program() self.check_program(program) val_graph = IrGraph(core.Graph(program.desc), for_test=False) if not for_ci: val_marked_nodes = set() for op in val_graph.all_op_nodes(): if op.name().find('quantize') > -1: val_marked_nodes.add(op) val_graph.draw('.', 'val_fc_' + activation_quant_type, val_marked_nodes)
def residual_block_quant(self, quantizable_op_type, skip_pattern=None, for_ci=True): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): loss = quant_dequant_residual_block(2, skip_pattern) opt = fluid.optimizer.Adam(learning_rate=0.001) opt.minimize(loss) place = fluid.CPUPlace() graph = IrGraph(core.Graph(main.desc), for_test=False) add_quant_dequant_pass = AddQuantDequantPass( scope=fluid.global_scope(), place=place, skip_pattern=skip_pattern, quantizable_op_type=quantizable_op_type) add_quant_dequant_pass.apply(graph) if not for_ci: marked_nodes = set() for op in graph.all_op_nodes(): if op.name().find('quant') > -1: marked_nodes.add(op) graph.draw('.', 'add_quant_dequant_graph', marked_nodes) self.check_graph(graph, skip_pattern) program = graph.to_program() val_graph = IrGraph(core.Graph(program.desc), for_test=False) if not for_ci: val_marked_nodes = set() for op in val_graph.all_op_nodes(): if op.name().find('quant') > -1: val_marked_nodes.add(op) val_graph.draw('.', 'val_add_quant_dequant_graph', val_marked_nodes)
def residual_block_quant(self, quant_type): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): loss = residual_block(2) opt = fluid.optimizer.Adam(learning_rate=0.001) opt.minimize(loss) place = fluid.CPUPlace() exe = fluid.Executor(place) graph = IrGraph(core.Graph(main.desc), for_test=False) transform_pass = QuantizationTransformPass( scope=fluid.global_scope(), place=place, activation_quantize_type=quant_type) transform_pass.apply(graph) marked_nodes = set() for op in graph.all_op_nodes(): if op.name().find('quantize') > -1: marked_nodes.add(op) graph.draw('.', 'quantize_residual_' + quant_type, marked_nodes) program = graph.to_program() self.check_program(transform_pass, program) val_graph = IrGraph(core.Graph(program.desc), for_test=False) val_marked_nodes = set() for op in val_graph.all_op_nodes(): if op.name().find('quantize') > -1: val_marked_nodes.add(op) val_graph.draw('.', 'val_residual_' + quant_type, val_marked_nodes)
def test_quant_update_activation(self): program = fluid.Program() with fluid.program_guard(program): self.prepare_program(program) graph = IrGraph(core.Graph(program.desc), for_test=True) quant2_int8_mkldnn_pass = Quant2Int8MkldnnPass( self.quantized_ops, _scope=self.scope, _place=self.place, _core=core, _debug=False) input_scale_tensor = quant2_int8_mkldnn_pass._convert_scale2tensor( np.array(self.scale).astype(np.float64)) output_scale_tensor = quant2_int8_mkldnn_pass._convert_scale2tensor( np.array(1. / self.scale * self.scale).astype(np.float64)) var_scale = { "input": (False, input_scale_tensor), "filter": (False, input_scale_tensor), "conv_output": (False, output_scale_tensor), } if core.avx_supported(): quant2_int8_mkldnn_pass._var_quant_scales = var_scale graph = quant2_int8_mkldnn_pass._propagate_scales(graph) graph = quant2_int8_mkldnn_pass._quantize_fp32_graph(graph) self.check_graph_after_pass(graph)
def quant_embedding(program, place, config, scope=None): """quantize lookup_table op parameters Args: program(fluid.Program): infer program scope(fluid.Scope): Scope records the mapping between variable names and variables, similar to brackets in programming languages. Usually users can use `fluid.global_scope() <https://www.paddlepaddle.org.cn/documentation/docs/zh/develop/api_cn/executor_cn/global_scope_cn.html>`_ . When ``None`` will use `fluid.global_scope() <https://www.paddlepaddle.org.cn/documentation/docs/zh/develop/api_cn/executor_cn/global_scope_cn.html>`_. Default : ``None``. place(fluid.CPUPlace or fluid.CUDAPlace): This parameter represents the executor run on which device. config(dict): config to quantize. The keys are 'params_name', 'quantize_type', \ 'quantize_bits', 'dtype', 'threshold'. \ ``params_name`` is parameter name to quantize, must be set. ``quantize_type`` is quantize type, supported types are ['abs_max'], default is "abs_max". ``quantize_bits`` supported bits are [8] and default is 8. ``dtype`` is quantize dtype, supported dtype are ['int8'], default is 'int8'. ``threshold`` is threshold to clip tensor before quant. When threshold is not set, \ tensor will not be clipped. Returns: None """ assert isinstance(config, dict), "config must be dict" config = _merge_config(copy.deepcopy(default_config), config) scope = fluid.global_scope() if scope is None else scope graph = IrGraph(core.Graph(program.desc), for_test=True) if config['quantize_type'] == 'abs_max': _quant_embedding_abs_max(graph, scope, place, config) return graph.to_program()
def transform_and_save_model(original_path, save_path, save_type): place = fluid.CPUPlace() exe = fluid.Executor(place) inference_scope = fluid.executor.global_scope() with fluid.scope_guard(inference_scope): if os.path.exists(os.path.join(original_path, '__model__')): [inference_program, feed_target_names, fetch_targets ] = fluid.io.load_inference_model(original_path, exe) else: [inference_program, feed_target_names, fetch_targets ] = fluid.io.load_inference_model(original_path, exe, 'model', 'params') quantized_ops = set(test_args.quantized_ops.split(',')) transform_to_mkldnn_int8_pass = Qat2Int8MkldnnPass( quantized_ops, _scope=inference_scope, _place=place, _core=core) graph = IrGraph(core.Graph(inference_program.desc), for_test=True) if save_type == 'FP32': graph = transform_to_mkldnn_int8_pass.apply_fp32(graph) elif save_type == 'INT8': graph = transform_to_mkldnn_int8_pass.apply(graph) inference_program = graph.to_program() with fluid.scope_guard(inference_scope): fluid.io.save_inference_model(save_path, feed_target_names, fetch_targets, exe, inference_program) print( "Success! Transformed QAT_{0} model can be found at {1}\n".format( save_type, save_path))
def test_dequantize_op_weights(self): program = fluid.Program() with fluid.program_guard(program): self.prepare_program_mul(program) graph = IrGraph(core.Graph(program.desc), for_test=True) op_node = "" for op in graph.all_op_nodes(): if op.op().type() == self.op_name(): op_node = op break assert op_node != "", "op of type %s not found" % self.op_name() qpass = Quant2Int8MkldnnPass(self.quantized_ops, _scope=self.scope, _place=self.place, _core=core, _debug=False) qpass._weight_thresholds["mul_output"] = self.mul_output_scale param = self.scope.var("mul_weights").get_tensor() param.set(self.variables_mul["mul_weights"], self.place) qpass._dequantize_op_weights(graph, op_node, "Y", "Out") assert np.allclose( self.scope.find_var("mul_weights").get_tensor(), [[1. / 127., 2. / 127., 3. / 127., 4. / 127., 5. / 127.], [1. / 127., 2. / 127., 3. / 127., 4. / 127., 5. / 127.], [1. / 127., 2. / 127., 3. / 127., 4. / 127., 5. / 127.]]) param = self.scope.var("mul_weights").get_tensor() param.set(self.variables_mul["mul_weights_bad"], self.place) with self.assertRaises(ValueError): qpass._dequantize_op_weights(graph, op_node, "Y", "Out")
def quant_embedding(program, place, config, scope=None): """ quant lookup_table op parameters Args: program(fluid.Program): infer program scope(fluid.Scope): the scope to store var, when is None will use fluid.global_scope() place(fluid.CPUPlace or fluid.CUDAPlace): place config(dict): config to quant. The keys are 'params_name', 'quantize_type', \ 'quantize_bits', 'dtype', 'threshold'. \ 'params_name': parameter name to quant, must be set. 'quantize_type': quantize type, supported types are ['abs_max']. default is "abs_max". 'quantize_bits': quantize bits, supported bits are [8]. default is 8. 'dtype': quantize dtype, supported dtype are ['int8']. default is 'int8'. 'threshold': threshold to clip tensor before quant. When threshold is not set, \ tensor will not be clipped. """ assert isinstance(config, dict), "config must be dict" config = _merge_config(copy.deepcopy(default_config), config) scope = fluid.global_scope() if scope is None else scope graph = IrGraph(core.Graph(program.desc), for_test=True) if config['quantize_type'] == 'abs_max': _quant_embedding_abs_max(graph, scope, place, config) return graph.to_program()
def test_dequantize_op_weights(self): program = fluid.Program() with fluid.program_guard(program): self.prepare_program_mul(program) graph = IrGraph(core.Graph(program.desc), for_test=True) for op in graph.all_op_nodes(): if op.op().type() == "mul": op_node = op break qpass = Quant2Int8MkldnnPass(self.quantized_ops, _scope=self.scope, _place=self.place, _core=core, _debug=False) qpass._weight_scales["mul_output"] = self.mul_output_scale param = self.scope.var("mul_weights").get_tensor() param.set(self.variables_mul["mul_weights"], self.place) qpass._dequantize_op_weights(graph, op_node, "Y", "Out") assert np.allclose( self.scope.find_var("mul_weights").get_tensor(), [[127, 63.5, 42.3333, 31.75, 25.4], [127, 63.5, 42.3333, 31.75, 25.4], [127, 63.5, 42.3333, 31.75, 25.4]]) param = self.scope.var("mul_weights").get_tensor() param.set(self.variables_mul["mul_weights_bad"], self.place) with self.assertRaises(ValueError): qpass._dequantize_op_weights(graph, op_node, "Y", "Out")
def transform_and_save_int8_model(original_path, save_path, ops_to_quantize='', op_ids_to_skip='', debug=False, quant_model_filename='', quant_params_filename='', save_model_filename="__model__", save_params_filename=None): place = fluid.CPUPlace() exe = fluid.Executor(place) inference_scope = fluid.executor.global_scope() with fluid.scope_guard(inference_scope): if not quant_model_filename: if os.path.exists(os.path.join(original_path, '__model__')): [inference_program, feed_target_names, fetch_targets ] = fluid.io.load_inference_model(original_path, exe) else: [inference_program, feed_target_names, fetch_targets ] = fluid.io.load_inference_model(original_path, exe, 'model', 'params') else: [inference_program, feed_target_names, fetch_targets ] = fluid.io.load_inference_model(original_path, exe, quant_model_filename, quant_params_filename) ops_to_quantize_set = set() print(ops_to_quantize) if len(ops_to_quantize) > 0: ops_to_quantize_set = set(ops_to_quantize.split(',')) op_ids_to_skip_set = set([-1]) print(op_ids_to_skip) if len(op_ids_to_skip) > 0: op_ids_to_skip_set = set(map(int, op_ids_to_skip.split(','))) graph = IrGraph(core.Graph(inference_program.desc), for_test=True) if (debug): graph.draw('.', 'quant_orig', graph.all_op_nodes()) transform_to_mkldnn_int8_pass = Quant2Int8MkldnnPass( ops_to_quantize_set, _op_ids_to_skip=op_ids_to_skip_set, _scope=inference_scope, _place=place, _core=core, _debug=debug) graph = transform_to_mkldnn_int8_pass.apply(graph) inference_program = graph.to_program() with fluid.scope_guard(inference_scope): fluid.io.save_inference_model(save_path, feed_target_names, fetch_targets, exe, inference_program, model_filename=save_model_filename, params_filename=save_params_filename) print( "Success! INT8 model obtained from the Quant model can be found at {}\n" .format(save_path))
def convert(program, place, config=None, scope=None, save_int8=False): """ convert quantized and well-trained ``program`` to final quantized ``program`` that can be used to save ``inference model``. Args: program(fluid.Program): quantized and well-trained ``test program``. place(fluid.CPUPlace or fluid.CUDAPlace): This parameter represents the executor run on which device. config(dict, optional): configs for convert. if set None, will use default config. It must be same with config that used in 'quant_aware'. Default: None. scope(fluid.Scope, optional): Scope records the mapping between variable names and variables, similar to brackets in programming languages. Usually users can use `fluid.global_scope <https://www.paddlepaddle.org.cn/documentation/docs/zh/develop/api_cn/executor_cn/global_scope_cn.html>`_. When ``None`` will use `fluid.global_scope() <https://www.paddlepaddle.org.cn/documentation/docs/zh/develop/api_cn/executor_cn/global_scope_cn.html>`_ . Default: ``None``. save_int8: Whether to return ``program`` which model parameters' dtype is ``int8``. This parameter can only be used to get model size. Default: ``False``. Returns: Tuple : freezed program which can be used for inference. when ``save_int8`` is False, return ``freezed_program(fluid.Program)``. when ``save_int8`` is True, return ``freezed_program(fluid.Program)`` and ``freezed_program_int8(fluid.Program)`` """ scope = fluid.global_scope() if not scope else scope if config is None: config = _quant_config_default else: assert isinstance(config, dict), "config must be dict" config = _parse_configs(config) _logger.info("convert config {}".format(config)) test_graph = IrGraph(core.Graph(program.desc), for_test=True) support_op_types = [] for op in config['quantize_op_types']: if op in QuantizationFreezePass._supported_quantizable_op_type: support_op_types.append(op) # Freeze the graph after training by adjusting the quantize # operators' order for the inference. freeze_pass = QuantizationFreezePass( scope=scope, place=place, weight_bits=config['weight_bits'], activation_bits=config['activation_bits'], weight_quantize_type=config['weight_quantize_type'], quantizable_op_type=support_op_types) freeze_pass.apply(test_graph) freezed_program = test_graph.to_program() if save_int8: convert_int8_pass = ConvertToInt8Pass( scope=fluid.global_scope(), place=place, quantizable_op_type=support_op_types) convert_int8_pass.apply(test_graph) freezed_program_int8 = test_graph.to_program() return freezed_program, freezed_program_int8 else: return freezed_program
def get_op_number(self, prog): graph = IrGraph(core.Graph(prog.desc), for_test=False) quant_op_nums = 0 op_nums = 0 for op in graph.all_op_nodes(): if op.name() in ['conv2d', 'depthwise_conv2d', 'mul']: op_nums += 1 elif 'fake_' in op.name(): quant_op_nums += 1 return op_nums, quant_op_nums
def test_quant_update_activation(self): program = fluid.Program() with fluid.program_guard(program): self.prepare_program(program) graph = IrGraph(core.Graph(program.desc), for_test=True) graph = self.remove_fuse_activation_attribute(graph) self.check_graph_before_pass(graph) quant2_int8_mkldnn_pass = Quant2Int8MkldnnPass(self.quantized_ops, _scope=self.scope, _place=self.place, _core=core, _debug=False) graph = quant2_int8_mkldnn_pass._update_activations(graph) self.check_graph_after_pass(graph)
def quant_embedding(program, place, config=None, scope=None): """quantize lookup_table op parameters Args: program(paddle.static.Program): infer program scope(paddle.static.Scope, optional): Scope records the mapping between variable names and variables, similar to brackets in programming languages. Usually users can use `paddle.static.global_scope() <https://www.paddlepaddle.org.cn/documentation/docs/zh/develop/api_cn/executor_cn/global_scope_cn.html>`_ . When ``None`` will use `paddle.static.global_scope() <https://www.paddlepaddle.org.cn/documentation/docs/zh/develop/api_cn/executor_cn/global_scope_cn.html>`_. Default : ``None``. place(paddle.CPUPlace or paddle.CUDAPlace): This parameter represents the executor run on which device. config(dict, optional): config to quantize. The keys are 'quantize_op_types'. For op in quantize_op_types, you can define 'quantize_type', \ 'quantize_bits', 'dtype', 'threshold'. \ ``quantize_type`` is quantize type, supported types are ['abs_max'], default is "abs_max". ``quantize_bits`` supported bits are [8] and default is 8. ``dtype`` is quantize dtype, supported dtype are ['int8'], default is 'int8'. ``threshold`` is threshold to clip tensor before quant. When threshold is not set, \ tensor will not be clipped. Returns: None """ config = config or {} config = _merge_config(copy.deepcopy(_default_config), config) scope = paddle.static.global_scope() if scope is None else scope graph = IrGraph(core.Graph(program.desc), for_test=True) quantize_params_map = {} all_op = graph.all_op_nodes() for op in all_op: if op.inputs == [] and op.outputs == []: continue op_type = op.name() if op_type in config['quantize_op_types']: weight_name = op.input('W')[0] if weight_name in quantize_params_map.values(): continue embedding_node = graph._find_node_by_name(op.inputs, op.input('W')[0]) for op_node in embedding_node.outputs: if op_node.name() == 'fused_embedding_seq_pool': _split_embedding_seq_pool(graph, op_node) if config[op_type]['quantize_type'] == 'abs_max': _quant_embedding_abs_max(graph, scope, place, config[op_type], weight_name, embedding_node) elif config[op_type]['quantize_type'] == 'log': _quant_embedding_log(graph, scope, place, config[op_type], weight_name, embedding_node) quantize_params_map[weight_name] = _get_quant_var_name(weight_name) for op in all_op: if op.name() == 'fused_embedding_seq_pool': graph.safe_remove_nodes(op) return graph.to_program()
def _predict(self, test_reader=None, model_path=None): place = fluid.CPUPlace() exe = fluid.Executor(place) inference_scope = fluid.executor.global_scope() with fluid.scope_guard(inference_scope): if os.path.exists(os.path.join(model_path, '__model__')): [inference_program, feed_target_names, fetch_targets ] = fluid.io.load_inference_model(model_path, exe) else: [inference_program, feed_target_names, fetch_targets ] = fluid.io.load_inference_model(model_path, exe, 'model', 'params') use_mkldnn = fluid.core.globals()["FLAGS_use_mkldnn"] if (use_mkldnn): graph = IrGraph(core.Graph(inference_program.desc), for_test=True) graph = self._transform_depthwise_conv(graph) inference_program = graph.to_program() dshape = [3, 224, 224] top1 = 0.0 top5 = 0.0 total_samples = 0 for batch_id, data in enumerate(test_reader()): if six.PY2: images = map(lambda x: x[0].reshape(dshape), data) if six.PY3: images = list(map(lambda x: x[0].reshape(dshape), data)) images = np.array(images).astype('float32') labels = np.array([x[1] for x in data]).astype("int64") labels = labels.reshape([-1, 1]) fluid.core.set_num_threads(int(os.environ['CPU_NUM_THREADS'])) out = exe.run(inference_program, feed={ feed_target_names[0]: images, feed_target_names[1]: labels }, fetch_list=fetch_targets) fluid.core.set_num_threads(1) top1 += np.sum(out[1]) * len(data) top5 += np.sum(out[2]) * len(data) total_samples += len(data) if (batch_id + 1) % 100 == 0: _logger.info( '{} images have been predicted'.format(total_samples)) return top1 / total_samples, top5 / total_samples
def generate_dot_for_model(model_path, save_graph_dir, save_graph_name): place = fluid.CPUPlace() exe = fluid.Executor(place) inference_scope = fluid.executor.global_scope() with fluid.scope_guard(inference_scope): if os.path.exists(os.path.join(model_path, '__model__')): [inference_program, feed_target_names, fetch_targets] = fluid.io.load_inference_model(model_path, exe) else: [inference_program, feed_target_names, fetch_targets] = fluid.io.load_inference_model(model_path, exe, 'model', 'params') graph = IrGraph(core.Graph(inference_program.desc), for_test=True) if not os.path.exists(save_graph_dir): os.makedirs(save_graph_dir) model_name = os.path.basename(os.path.normpath(save_graph_dir)) if save_graph_name is '': save_graph_name = model_name graph.draw(save_graph_dir, save_graph_name, graph.all_op_nodes()) print( "Success! Generated dot and pdf files for {0} model, that can be found at {1} named {2}.\n". format(model_name, save_graph_dir, save_graph_name))
def test_graph_functions(self): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): loss = residual_block(2) opt = fluid.optimizer.Adam(learning_rate=0.001) opt.minimize(loss) graph = IrGraph(core.Graph(main.desc), for_test=False) marked_nodes = set() for op in graph.all_op_nodes(): if op.name().find('conv2d') > -1: marked_nodes.add(op) graph.draw('.', 'residual', marked_nodes) self.assertFalse(graph.has_circle()) self.assertEqual(graph.graph_num(), 1) nodes = graph.topology_sort() self.assertEqual(len(nodes), len(graph.all_op_nodes())) nodes_map = graph.build_adjacency_list() self.assertEqual(len(nodes_map), len(graph.all_op_nodes())) nodes_num = len(graph.all_nodes()) graph.safe_remove_nodes(marked_nodes) self.assertEqual(len(graph.all_nodes()), nodes_num - len(marked_nodes))
def build_graph_with_sub_graph(self): def linear_fc(num): data = fluid.layers.data(name='image', shape=[1, 32, 32], dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') hidden = data for _ in six.moves.xrange(num): hidden = fluid.layers.fc(hidden, size=128, act='relu') loss = fluid.layers.cross_entropy(input=hidden, label=label) loss = fluid.layers.mean(loss) return loss main_program = Program() startup_program = Program() def true_func(): return linear_fc(3) def false_func(): return linear_fc(5) with program_guard(main_program, startup_program): x = layers.fill_constant(shape=[1], dtype='float32', value=0.1) y = layers.fill_constant(shape=[1], dtype='float32', value=0.23) pred = layers.less_than(y, x) out = layers.cond(pred, true_func, false_func) core_graph = core.Graph(main_program.desc) # We should create graph for test, otherwise it will throw a # error that it cannot find the node of "STEP_COUNTER" graph = IrGraph(core_graph, for_test=True) sub_graph = graph.get_sub_graph(0) all_sub_graphs = graph.all_sub_graphs( for_test=True) # same reason for subgraph # Should return graph and sub_graphs at the same time. If only return sub_graph, the graph will # be destructed and the sub_graphs will be empty. return graph, all_sub_graphs
def _predict(self, test_reader=None, model_path=None, batch_size=1, batch_num=1, skip_batch_num=0, transform_to_int8=False): place = fluid.CPUPlace() exe = fluid.Executor(place) inference_scope = fluid.executor.global_scope() with fluid.scope_guard(inference_scope): if os.path.exists(os.path.join(model_path, '__model__')): [inference_program, feed_target_names, fetch_targets ] = fluid.io.load_inference_model(model_path, exe) else: [inference_program, feed_target_names, fetch_targets ] = fluid.io.load_inference_model(model_path, exe, 'model', 'params') graph = IrGraph(core.Graph(inference_program.desc), for_test=True) if (self._debug): graph.draw('.', 'quant_orig', graph.all_op_nodes()) if (transform_to_int8): mkldnn_int8_pass = QuantInt8MkldnnPass(_scope=inference_scope, _place=place) graph = mkldnn_int8_pass.apply(graph) else: graph = self._prepare_for_fp32_mkldnn(graph) inference_program = graph.to_program() dshape = [3, 224, 224] outputs = [] infer_accs1 = [] infer_accs5 = [] fpses = [] batch_times = [] total_samples = 0 iters = 0 infer_start_time = time.time() for data in test_reader(): if batch_num > 0 and iters >= batch_num: break if iters == skip_batch_num: total_samples = 0 infer_start_time = time.time() if six.PY2: images = map(lambda x: x[0].reshape(dshape), data) if six.PY3: images = list(map(lambda x: x[0].reshape(dshape), data)) images = np.array(images).astype('float32') labels = np.array([x[1] for x in data]).astype('int64') start = time.time() out = exe.run(inference_program, feed={feed_target_names[0]: images}, fetch_list=fetch_targets) batch_time = (time.time() - start) * 1000 # in miliseconds outputs.append(out[0]) batch_acc1, batch_acc5 = self._get_batch_accuracy( out[0], labels) infer_accs1.append(batch_acc1) infer_accs5.append(batch_acc5) samples = len(data) total_samples += samples batch_times.append(batch_time) fps = samples / batch_time * 1000 fpses.append(fps) iters += 1 appx = ' (warm-up)' if iters <= skip_batch_num else '' _logger.info('batch {0}{5}, acc1: {1:.4f}, acc5: {2:.4f}, ' 'latency: {3:.4f} ms, fps: {4:.2f}'.format( iters, batch_acc1, batch_acc5, batch_time / batch_size, fps, appx)) # Postprocess benchmark data batch_latencies = batch_times[skip_batch_num:] batch_latency_avg = np.average(batch_latencies) latency_avg = batch_latency_avg / batch_size fpses = fpses[skip_batch_num:] fps_avg = np.average(fpses) infer_total_time = time.time() - infer_start_time acc1_avg = np.mean(infer_accs1) acc5_avg = np.mean(infer_accs5) _logger.info( 'Total inference run time: {:.2f} s'.format(infer_total_time)) return outputs, acc1_avg, acc5_avg, fps_avg, latency_avg
def graph_apis(self, use_cuda=False, for_ci=True): main = fluid.Program() startup = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, startup): feeds, loss = conv_block() opt = fluid.optimizer.Adam(learning_rate=0.001) opt.minimize(loss) graph = IrGraph(core.Graph(main.desc), for_test=False) backup_graph = graph.clone() self.assertEqual(len(graph.all_nodes()), len(backup_graph.all_nodes())) build_strategy = fluid.BuildStrategy() build_strategy.memory_optimize = False build_strategy.enable_inplace = False origin_binary = fluid.CompiledProgram(graph.graph).with_data_parallel( loss_name=loss.name, build_strategy=build_strategy) backup_binary = fluid.CompiledProgram( backup_graph.graph).with_data_parallel( loss_name=loss.name, build_strategy=build_strategy) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup) iters = 5 batch_size = 8 train_reader = paddle.batch( paddle.dataset.mnist.train(), batch_size=batch_size) feeder = fluid.DataFeeder(feed_list=feeds, place=place) def _train(binary): for _ in range(iters): data = next(train_reader()) loss_v = exe.run(binary, feed=feeder.feed(data), fetch_list=[loss.name]) if not for_ci: print('{}: {}'.format('loss', loss_v)) _train(origin_binary) _train(backup_binary) checkponit_dir = "checkpoint_gpu" if use_cuda else "checkpoint_cpu" def _set_zero(var_name, scope, place): var = scope.find_var(var_name).get_tensor() var_array = np.zeros(var._get_dims()).astype("float32") var.set(var_array, place) sum_before = np.sum( np.array(fluid.global_scope().find_var('conv2d_1.w_0').get_tensor( ))) fluid.io._save_persistable_nodes(exe, checkponit_dir, graph) _set_zero('conv2d_1.w_0', fluid.global_scope(), place) set_after = np.sum( np.array(fluid.global_scope().find_var('conv2d_1.w_0').get_tensor( ))) self.assertEqual(set_after, 0) fluid.io._load_persistable_nodes(exe, checkponit_dir, graph) sum_after = np.sum( np.array(fluid.global_scope().find_var('conv2d_1.w_0').get_tensor( ))) self.assertEqual(sum_before, sum_after) marked_nodes = set() for op in graph.all_op_nodes(): if op.name().find('conv2d') > -1: marked_nodes.add(op) if not for_ci: graph.draw('.', 'residual', marked_nodes) backup_marked_nodes = set() for op in backup_graph.all_op_nodes(): if op.name().find('conv2d') > -1: backup_marked_nodes.add(op) backup_graph.draw('./origin', 'backup', backup_marked_nodes) self.assertFalse(graph.has_circle()) self.assertEqual(graph.graph_num(), 1) nodes = graph.topology_sort() self.assertEqual(len(nodes), len(graph.all_op_nodes())) nodes_map = graph.build_adjacency_list() self.assertEqual(len(nodes_map), len(graph.all_op_nodes())) nodes_num = len(graph.all_nodes()) graph.safe_remove_nodes(marked_nodes) self.assertEqual(len(graph.all_nodes()), nodes_num - len(marked_nodes))
def check_output_with_option(self, use_gpu, atol=1e-5, flatten=False, quant=False, rtol=1e-5): ''' Check whether calculating on CPU and GPU, enable TensorRT or disable TensorRT, enable MKLDNN or disable MKLDNN are all the same. ''' place = fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace() executor = fluid.Executor(place) scope = fluid.Scope() device = "GPU" if use_gpu else "CPU" with fluid.scope_guard(scope): executor.run(self.startup_program) executor.run(self.test_startup_program) main_graph = IrGraph(core.Graph(self.main_program.desc), for_test=False) test_graph = IrGraph(core.Graph(self.test_main_program.desc), for_test=True) transform_pass = QuantizationTransformPass( scope=scope, place=place, activation_quantize_type=self.activation_quantize_type, weight_quantize_type=self.weight_quantize_type) transform_pass.apply(main_graph) transform_pass.apply(test_graph) add_quant_dequant_pass = AddQuantDequantPass(scope=scope, place=place) add_quant_dequant_pass.apply(main_graph) add_quant_dequant_pass.apply(test_graph) scale_training_pass = OutScaleForTrainingPass(scope=scope, place=place) scale_training_pass.apply(main_graph) build_strategy = fluid.BuildStrategy() build_strategy.memory_optimize = False build_strategy.enable_inplace = False build_strategy.fuse_all_reduce_ops = False binary = fluid.CompiledProgram(main_graph.graph) iters = 10 batch_size = 1 train_reader = paddle.batch(paddle.reader.shuffle( paddle.dataset.mnist.train(), buf_size=500), batch_size=batch_size) feeder = fluid.DataFeeder(feed_list=[self.data, self.label], place=place) with fluid.scope_guard(scope): for _ in range(iters): data = next(train_reader()) loss_v = executor.run(binary, feed=feeder.feed(data), fetch_list=[self.loss]) scale_inference_pass = OutScaleForInferencePass(scope=scope) scale_inference_pass.apply(test_graph) # Freeze graph for inference, but the weight of fc/conv is still float type. freeze_pass = QuantizationFreezePass( scope=scope, place=place, weight_quantize_type=self.weight_quantize_type) freeze_pass.apply(test_graph) self.main_program = test_graph.to_program() with fluid.scope_guard(scope): self.main_program = self._normalize_program( self.main_program, self.data, self.fetch_list) self._save_models(self.path, list(self.feeds.keys()), self.fetch_list, executor, self.main_program, scope) paddle_outs = self._get_paddle_outs(self.feeds, self.fetch_list, executor, self.main_program, scope) inference_outs = self._get_inference_outs( self._get_analysis_config(use_gpu=use_gpu)) # Check whether the results calculated on CPU and on GPU are the same. self.assertTrue( len(paddle_outs) == len(inference_outs), "The number of outputs is different between inference and training forward at {}" .format(device)) for out, inference_out in zip(paddle_outs, inference_outs): paddle_out = np.array(out) if flatten: paddle_out = paddle_out.flatten() inference_out = inference_out.flatten() self.assertTrue( np.allclose(paddle_out, inference_out, atol=atol), "Output has diff between inference and training forward at {} " .format(device)) # Check whether the trt results and the GPU results are the same. if use_gpu and self.enable_trt: tensorrt_outputs = self._get_inference_outs( self._get_analysis_config(use_gpu=use_gpu, use_trt=self.enable_trt)) if self.trt_parameters.use_static: #deserialize tensorrt_outputs = self._get_inference_outs( self._get_analysis_config(use_gpu=use_gpu, use_trt=self.enable_trt)) self.assertTrue( len(tensorrt_outputs) == len(paddle_outs), "The number of outputs is different between GPU and TensorRT. " ) for paddle_out, tensorrt_output in zip(paddle_outs, tensorrt_outputs): paddle_out = np.array(paddle_out) if flatten: paddle_out = paddle_out.flatten() tensorrt_output = tensorrt_output.flatten() self.assertTrue( np.allclose(paddle_out, tensorrt_output, rtol=rtol, atol=atol), "Output has diff between GPU and TensorRT. ") # Check whether the mkldnn results and the CPU results are the same. if (not use_gpu) and self.enable_mkldnn: mkldnn_outputs = self._get_inference_outs( self._get_analysis_config(use_gpu=use_gpu, use_mkldnn=self.enable_mkldnn)) self.assertTrue( len(paddle_outs) == len(mkldnn_outputs), "The number of outputs is different between CPU and MKLDNN. ") if self.enable_mkldnn_bfloat16: atol = 0.01 for paddle_out, mkldnn_output in zip(paddle_outs, mkldnn_outputs): self.assertTrue( np.allclose(np.array(paddle_out), mkldnn_output, atol=atol), "Output has diff between CPU and MKLDNN. ")
def quantization_scale(self, use_cuda, seed, activation_quant_type, weight_quant_type='abs_max', for_ci=False): def build_program(main, startup, is_test): main.random_seed = seed startup.random_seed = seed with fluid.unique_name.guard(): with fluid.program_guard(main, startup): img = fluid.layers.data(name='image', shape=[1, 28, 28], dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') loss = residual_block(img, label, 1) if not is_test: opt = fluid.optimizer.Adam(learning_rate=0.0001) opt.minimize(loss) return [img, label], loss random.seed(0) np.random.seed(0) main = fluid.Program() startup = fluid.Program() test_program = fluid.Program() feeds, loss = build_program(main, startup, False) build_program(test_program, startup, True) test_program = test_program.clone(for_test=True) main_graph = IrGraph(core.Graph(main.desc), for_test=False) test_graph = IrGraph(core.Graph(test_program.desc), for_test=True) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) scope = fluid.Scope() with fluid.scope_guard(scope): exe.run(startup) transform_pass = QuantizationTransformPass( scope=scope, place=place, activation_quantize_type=activation_quant_type, weight_quantize_type=weight_quant_type) transform_pass.apply(main_graph) transform_pass.apply(test_graph) add_quant_dequant_pass = AddQuantDequantPass(scope=scope, place=place) add_quant_dequant_pass.apply(main_graph) add_quant_dequant_pass.apply(test_graph) scale_training_pass = OutScaleForTrainingPass(scope=scope, place=place) scale_training_pass.apply(main_graph) dev_name = '_gpu' if use_cuda else '_cpu' if not for_ci: marked_nodes = set() for op in main_graph.all_op_nodes(): if op.name().find('quantize') > -1: marked_nodes.add(op) main_graph.draw('.', 'main_scale' + dev_name, marked_nodes) marked_nodes = set() for op in test_graph.all_op_nodes(): if op.name().find('quantize') > -1: marked_nodes.add(op) test_graph.draw('.', 'test_scale' + dev_name, marked_nodes) build_strategy = fluid.BuildStrategy() build_strategy.memory_optimize = False build_strategy.enable_inplace = False build_strategy.fuse_all_reduce_ops = False binary = fluid.CompiledProgram(main_graph.graph).with_data_parallel( loss_name=loss.name, build_strategy=build_strategy) iters = 5 batch_size = 8 train_reader = paddle.batch(paddle.reader.shuffle( paddle.dataset.mnist.train(), buf_size=500), batch_size=batch_size) feeder = fluid.DataFeeder(feed_list=feeds, place=place) with fluid.scope_guard(scope): for _ in range(iters): data = next(train_reader()) loss_v = exe.run(binary, feed=feeder.feed(data), fetch_list=[loss]) if not for_ci: print('{}: {}'.format('loss' + dev_name, loss_v)) scale_inference_pass = OutScaleForInferencePass(scope=scope) scale_inference_pass.apply(test_graph) # Freeze graph for inference, but the weight of fc/conv is still float type. freeze_pass = QuantizationFreezePass( scope=scope, place=place, weight_quantize_type=weight_quant_type) freeze_pass.apply(test_graph) server_program = test_graph.to_program() if not for_ci: marked_nodes = set() for op in test_graph.all_op_nodes(): if op.name().find('quantize') > -1: marked_nodes.add(op) test_graph.draw('.', 'quant_scale' + dev_name, marked_nodes) with open('quant_scale_model' + dev_name + '.txt', 'w') as f: f.write(str(server_program)) with fluid.scope_guard(scope): fluid.io.save_inference_model('quant_scale_model' + dev_name, ['image', 'label'], [loss], exe, server_program)
def check_output_with_option(self, use_gpu, atol=1e-5, flatten=False, quant=False): ''' Check whether calculating on CPU and GPU, enable TensorRT or disable TensorRT, enable MKLDNN or disable MKLDNN are all the same. ''' place = fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace() executor = fluid.Executor(place) scope = fluid.Scope() device = "GPU" if use_gpu else "CPU" with fluid.scope_guard(scope): executor.run(self.startup_program) if quant: main_graph = IrGraph(core.Graph(self.main_program.desc), for_test=True) transform_pass = QuantizationTransformPass( scope=scope, place=place, activation_quantize_type=self.activation_quant_type, weight_quantize_type=self.weight_quant_type, quantizable_op_type=[ 'conv2d', 'mul', 'depthwise_conv2d', 'conv2d_transpose' ]) transform_pass.apply(main_graph) weight_scale_map = { "conv2d": "conv2d_0.w_0.scale", "mul": "fc_0.w_0.scale" } weight_scale_tensor = scope.var( weight_scale_map[self.quantized_op_type]).get_tensor() weight_scale = np.ones(self.channels).astype("float32") weight_scale_tensor.set(weight_scale, place) op_nodes = main_graph.all_op_nodes() for op_node in op_nodes: if op_node.name() in [self.quantized_op_type, "relu"]: op_node.op()._set_attr("out_threshold", 0.5) with fluid.scope_guard(scope): executor.run(program=self.main_program, feed=self.feeds, fetch_list=self.fetch_list) freeze_pass = QuantizationFreezePass( scope=scope, place=place, weight_quantize_type=self.weight_quant_type) freeze_pass.apply(main_graph) self.main_program = main_graph.to_program() outs = self._save_models(executor, self.main_program, scope) analysis_outputs = self._get_analysis_outputs( self._get_analysis_config(use_gpu=use_gpu)) # Check whether the results calculated on CPU and on GPU are the same. self.assertTrue( len(outs) == len(analysis_outputs), "The number of outputs is different between inference and training forward at {}" .format(device)) for out, analysis_output in zip(outs, analysis_outputs): out = np.array(out) if flatten: out = out.flatten() analysis_output = analysis_output.flatten() self.assertTrue( np.allclose(out, analysis_output, atol=atol), "Output has diff between inference and training forward at {} " .format(device)) # Check whether the trt results and the GPU results are the same. if use_gpu and self.enable_trt: tensorrt_outputs = self._get_analysis_outputs( self._get_analysis_config(use_gpu=use_gpu, use_trt=self.enable_trt)) if self.trt_parameters.use_static: #deserialize tensorrt_outputs = self._get_analysis_outputs( self._get_analysis_config(use_gpu=use_gpu, use_trt=self.enable_trt)) self.assertTrue( len(tensorrt_outputs) == len(outs), "The number of outputs is different between GPU and TensorRT. " ) for out, tensorrt_output in zip(outs, tensorrt_outputs): out = np.array(out) if flatten: out = out.flatten() tensorrt_output = tensorrt_output.flatten() self.assertTrue(np.allclose(out, tensorrt_output, atol=atol), "Output has diff between GPU and TensorRT. ") # Check whether the mkldnn results and the CPU results are the same. if (not use_gpu) and self.enable_mkldnn: mkldnn_outputs = self._get_analysis_outputs( self._get_analysis_config(use_gpu=use_gpu, use_mkldnn=self.enable_mkldnn)) self.assertTrue( len(outs) == len(mkldnn_outputs), "The number of outputs is different between CPU and MKLDNN. ") if self.enable_mkldnn_bfloat16: atol = 0.01 for out, mkldnn_output in zip(outs, mkldnn_outputs): self.assertTrue( np.allclose(np.array(out), mkldnn_output, atol=atol), "Output has diff between CPU and MKLDNN. ")
def test_qat_acc(self): def _build_static_lenet(main, startup, is_test=False, seed=1000): with fluid.unique_name.guard(): with fluid.program_guard(main, startup): main.random_seed = seed startup.random_seed = seed img = fluid.layers.data( name='image', shape=[1, 28, 28], dtype='float32') label = fluid.layers.data( name='label', shape=[1], dtype='int64') prediction = StaticLenet(img) if not is_test: loss = fluid.layers.cross_entropy( input=prediction, label=label) avg_loss = fluid.layers.mean(loss) else: avg_loss = prediction return img, label, avg_loss reader = paddle.batch( paddle.dataset.mnist.test(), batch_size=32, drop_last=True) weight_quantize_type = 'abs_max' activation_quant_type = 'moving_average_abs_max' param_init_map = {} seed = 1000 lr = 0.001 # imperative train _logger.info( "--------------------------dynamic graph qat--------------------------" ) imperative_qat = ImperativeQuantAware( weight_quantize_type=weight_quantize_type, activation_quantize_type=activation_quant_type, quantizable_layer_type=[ 'Conv2D', 'Linear', 'ReLU', 'LeakyReLU', 'ReLU6', 'Tanh', 'Swish' ]) with fluid.dygraph.guard(): np.random.seed(seed) fluid.default_main_program().random_seed = seed fluid.default_startup_program().random_seed = seed lenet = ImperativeLenet() fixed_state = {} for name, param in lenet.named_parameters(): p_shape = param.numpy().shape p_value = param.numpy() if name.endswith("bias"): value = np.zeros_like(p_value).astype('float32') else: value = np.random.normal( loc=0.0, scale=0.01, size=np.product(p_shape)).reshape( p_shape).astype('float32') fixed_state[name] = value param_init_map[param.name] = value lenet.set_dict(fixed_state) imperative_qat.quantize(lenet) adam = AdamOptimizer( learning_rate=lr, parameter_list=lenet.parameters()) dynamic_loss_rec = [] lenet.train() for batch_id, data in enumerate(reader()): x_data = np.array([x[0].reshape(1, 28, 28) for x in data]).astype('float32') y_data = np.array( [x[1] for x in data]).astype('int64').reshape(-1, 1) img = fluid.dygraph.to_variable(x_data) label = fluid.dygraph.to_variable(y_data) out = lenet(img) loss = fluid.layers.cross_entropy(out, label) avg_loss = fluid.layers.mean(loss) avg_loss.backward() adam.minimize(avg_loss) lenet.clear_gradients() dynamic_loss_rec.append(avg_loss.numpy()[0]) if batch_id % 100 == 0: _logger.info('{}: {}'.format('loss', avg_loss.numpy())) if batch_id > 500: break lenet.eval() paddle.jit.save( layer=lenet, path="./dynamic_mnist/model", input_spec=[ paddle.static.InputSpec( shape=[None, 1, 28, 28], dtype='float32') ]) # static graph train _logger.info( "--------------------------static graph qat--------------------------" ) static_loss_rec = [] if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) else: place = core.CPUPlace() exe = fluid.Executor(place) main = fluid.Program() infer = fluid.Program() startup = fluid.Program() static_img, static_label, static_loss = _build_static_lenet( main, startup, False, seed) infer_img, _, infer_pre = _build_static_lenet(infer, startup, True, seed) with fluid.unique_name.guard(): with fluid.program_guard(main, startup): opt = AdamOptimizer(learning_rate=lr) opt.minimize(static_loss) scope = core.Scope() with fluid.scope_guard(scope): exe.run(startup) for param in main.all_parameters(): param_tensor = scope.var(param.name).get_tensor() param_tensor.set(param_init_map[param.name], place) main_graph = IrGraph(core.Graph(main.desc), for_test=False) infer_graph = IrGraph(core.Graph(infer.desc), for_test=True) transform_pass = QuantizationTransformPass( scope=scope, place=place, activation_quantize_type=activation_quant_type, weight_quantize_type=weight_quantize_type, quantizable_op_type=['conv2d', 'depthwise_conv2d', 'mul']) add_quant_dequant_pass = AddQuantDequantPass( scope=scope, place=place, quantizable_op_type=[ 'relu', 'leaky_relu', 'relu6', 'tanh', 'swish' ]) transform_pass.apply(main_graph) transform_pass.apply(infer_graph) add_quant_dequant_pass.apply(main_graph) add_quant_dequant_pass.apply(infer_graph) build_strategy = fluid.BuildStrategy() build_strategy.fuse_all_reduce_ops = False binary = fluid.CompiledProgram(main_graph.graph).with_data_parallel( loss_name=static_loss.name, build_strategy=build_strategy) feeder = fluid.DataFeeder( feed_list=[static_img, static_label], place=place) with fluid.scope_guard(scope): for batch_id, data in enumerate(reader()): loss_v, = exe.run(binary, feed=feeder.feed(data), fetch_list=[static_loss]) static_loss_rec.append(loss_v[0]) if batch_id % 100 == 0: _logger.info('{}: {}'.format('loss', loss_v)) save_program = infer_graph.to_program() with fluid.scope_guard(scope): fluid.io.save_inference_model("./static_mnist", [infer_img.name], [infer_pre], exe, save_program) rtol = 1e-08 atol = 1e-10 for i, (loss_d, loss_s) in enumerate(zip(dynamic_loss_rec, static_loss_rec)): diff = np.abs(loss_d - loss_s) if diff > (atol + rtol * np.abs(loss_s)): _logger.info( "diff({}) at {}, dynamic loss = {}, static loss = {}". format(diff, i, loss_d, loss_s)) break self.assertTrue( np.allclose( np.array(dynamic_loss_rec), np.array(static_loss_rec), rtol=rtol, atol=atol, equal_nan=True), msg='Failed to do the imperative qat.')
def freeze_graph(self, use_cuda, seed, activation_quant_type, weight_quant_type='abs_max', for_ci=True, quant_skip_pattern='skip_quant'): def build_program(main, startup, is_test): main.random_seed = seed startup.random_seed = seed with fluid.unique_name.guard(): with fluid.program_guard(main, startup): img = fluid.layers.data( name='image', shape=[1, 28, 28], dtype='float32') label = fluid.layers.data( name='label', shape=[1], dtype='int64') loss = conv_net(img, label, quant_skip_pattern) if not is_test: opt = fluid.optimizer.Adam(learning_rate=0.001) opt.minimize(loss) return [img, label], loss random.seed(0) np.random.seed(0) main = fluid.Program() startup = fluid.Program() test_program = fluid.Program() feeds, loss = build_program(main, startup, False) build_program(test_program, startup, True) test_program = test_program.clone(for_test=True) main_graph = IrGraph(core.Graph(main.desc), for_test=False) test_graph = IrGraph(core.Graph(test_program.desc), for_test=True) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) scope = fluid.Scope() with fluid.scope_guard(scope): exe.run(startup) transform_pass = QuantizationTransformPass( scope=scope, place=place, activation_quantize_type=activation_quant_type, weight_quantize_type=weight_quant_type, skip_pattern=quant_skip_pattern) transform_pass.apply(main_graph) transform_pass.apply(test_graph) dev_name = '_gpu_' if use_cuda else '_cpu_' if not for_ci: marked_nodes = set() for op in main_graph.all_op_nodes(): if op.name().find('quantize') > -1: marked_nodes.add(op) main_graph.draw('.', 'main' + dev_name + activation_quant_type + '_' + weight_quant_type, marked_nodes) marked_nodes = set() for op in test_graph.all_op_nodes(): if op.name().find('quantize') > -1: marked_nodes.add(op) test_graph.draw('.', 'test' + dev_name + activation_quant_type + '_' + weight_quant_type, marked_nodes) build_strategy = fluid.BuildStrategy() build_strategy.memory_optimize = False build_strategy.enable_inplace = False binary = fluid.CompiledProgram(main_graph.graph).with_data_parallel( loss_name=loss.name, build_strategy=build_strategy) quantized_test_program = test_graph.to_program() iters = 5 batch_size = 8 train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.mnist.train(), buf_size=500), batch_size=batch_size) test_reader = paddle.batch( paddle.dataset.mnist.test(), batch_size=batch_size) feeder = fluid.DataFeeder(feed_list=feeds, place=place) with fluid.scope_guard(scope): for _ in range(iters): data = next(train_reader()) loss_v = exe.run(binary, feed=feeder.feed(data), fetch_list=[loss]) if not for_ci: print('{}: {}'.format('loss' + dev_name + activation_quant_type + '_' + weight_quant_type, loss_v)) test_data = next(test_reader()) with fluid.program_guard(quantized_test_program): w_var = fluid.framework._get_var('conv2d_1.w_0.quantized', quantized_test_program) # Testing with fluid.scope_guard(scope): test_loss1, w_quant = exe.run(program=quantized_test_program, feed=feeder.feed(test_data), fetch_list=[loss, w_var]) # Freeze graph for inference, but the weight of fc/conv is still float type. freeze_pass = QuantizationFreezePass( scope=scope, place=place, weight_quantize_type=weight_quant_type) freeze_pass.apply(test_graph) if not for_ci: marked_nodes = set() for op in test_graph.all_op_nodes(): if op.name().find('quantize') > -1: marked_nodes.add(op) test_graph.draw('.', 'test_freeze' + dev_name + activation_quant_type + '_' + weight_quant_type, marked_nodes) server_program = test_graph.to_program() with fluid.scope_guard(scope): test_loss2, = exe.run(program=server_program, feed=feeder.feed(test_data), fetch_list=[loss]) self.assertAlmostEqual(test_loss1, test_loss2, delta=5e-3) if not for_ci: print( '{}: {}'.format('test_loss1' + dev_name + activation_quant_type + '_' + weight_quant_type, test_loss1)) print( '{}: {}'.format('test_loss2' + dev_name + activation_quant_type + '_' + weight_quant_type, test_loss2)) w_freeze = np.array(scope.find_var('conv2d_1.w_0').get_tensor()) # Maybe failed, this is due to the calculation precision # self.assertAlmostEqual(np.sum(w_freeze), np.sum(w_quant)) if not for_ci: print('{}: {}'.format('w_freeze' + dev_name + activation_quant_type + '_' + weight_quant_type, np.sum(w_freeze))) print('{}: {}'.format('w_quant' + dev_name + activation_quant_type + '_' + weight_quant_type, np.sum(w_quant))) # Convert parameter to 8-bit. convert_int8_pass = ConvertToInt8Pass(scope=scope, place=place) convert_int8_pass.apply(test_graph) if not for_ci: marked_nodes = set() for op in test_graph.all_op_nodes(): if op.name().find('quantize') > -1: marked_nodes.add(op) test_graph.draw('.', 'test_int8' + dev_name + activation_quant_type + '_' + weight_quant_type, marked_nodes) server_program_int8 = test_graph.to_program() # Save the 8-bit parameter and model file. with fluid.scope_guard(scope): fluid.io.save_inference_model( 'server_int8' + dev_name + activation_quant_type + '_' + weight_quant_type, ['image', 'label'], [loss], exe, server_program_int8) # Test whether the 8-bit parameter and model file can be loaded successfully. [infer, feed, fetch] = fluid.io.load_inference_model( 'server_int8' + dev_name + activation_quant_type + '_' + weight_quant_type, exe) # Check the loaded 8-bit weight. w_8bit = np.array(scope.find_var('conv2d_1.w_0.int8').get_tensor()) self.assertEqual(w_8bit.dtype, np.int8) self.assertEqual(np.sum(w_8bit), np.sum(w_freeze)) if not for_ci: print('{}: {}'.format('w_8bit' + dev_name + activation_quant_type + '_' + weight_quant_type, np.sum(w_8bit))) print('{}: {}'.format('w_freeze' + dev_name + activation_quant_type + '_' + weight_quant_type, np.sum(w_freeze))) mobile_pass = TransformForMobilePass() mobile_pass.apply(test_graph) if not for_ci: marked_nodes = set() for op in test_graph.all_op_nodes(): if op.name().find('quantize') > -1: marked_nodes.add(op) test_graph.draw('.', 'test_mobile' + dev_name + activation_quant_type + '_' + weight_quant_type, marked_nodes) mobile_program = test_graph.to_program() with fluid.scope_guard(scope): fluid.io.save_inference_model( 'mobile_int8' + dev_name + activation_quant_type + '_' + weight_quant_type, ['image', 'label'], [loss], exe, mobile_program)
def quantization_scale(self, use_cuda, seed, activation_quant_type, weight_quant_type='abs_max', for_ci=False, act_preprocess_func=None, weight_preprocess_func=None, act_quantize_func=None, weight_quantize_func=None): def build_program(main, startup, is_test): main.random_seed = seed startup.random_seed = seed with fluid.unique_name.guard(): with fluid.program_guard(main, startup): img = fluid.layers.data(name='image', shape=[1, 28, 28], dtype='float32') img.stop_gradient = False label = fluid.layers.data(name='label', shape=[1], dtype='int64') loss = conv_net(img, label) if not is_test: opt = fluid.optimizer.SGD(learning_rate=0.0001) opt.minimize(loss) return [img, label], loss def get_optimizer(): return fluid.optimizer.MomentumOptimizer(0.0001, 0.9) def load_dict(): with open('mapping_table_for_saving_inference_model', 'r') as file: data = file.read() data = json.loads(data) return data def save_dict(Dict): with open('mapping_table_for_saving_inference_model', 'w') as file: file.write(json.dumps(Dict)) random.seed(0) np.random.seed(0) main = fluid.Program() startup = fluid.Program() test_program = fluid.Program() feeds, loss = build_program(main, startup, False) build_program(test_program, startup, True) test_program = test_program.clone(for_test=True) main_graph = IrGraph(core.Graph(main.desc), for_test=False) test_graph = IrGraph(core.Graph(test_program.desc), for_test=True) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) scope = fluid.Scope() with fluid.scope_guard(scope): exe.run(startup) train_transform_pass = QuantizationTransformPass( scope=scope, place=place, activation_quantize_type=activation_quant_type, weight_quantize_type=weight_quant_type, act_preprocess_func=act_preprocess_func, weight_preprocess_func=weight_preprocess_func, act_quantize_func=act_quantize_func, weight_quantize_func=weight_quantize_func, optimizer_func=get_optimizer, executor=exe) train_transform_pass.apply(main_graph) test_transform_pass = QuantizationTransformPass( scope=scope, place=place, activation_quantize_type=activation_quant_type, weight_quantize_type=weight_quant_type, act_preprocess_func=act_preprocess_func, weight_preprocess_func=weight_preprocess_func, act_quantize_func=act_quantize_func, weight_quantize_func=weight_quantize_func, optimizer_func=get_optimizer, executor=exe) test_transform_pass.apply(test_graph) save_dict(test_graph.out_node_mapping_table) add_quant_dequant_pass = AddQuantDequantPass(scope=scope, place=place) add_quant_dequant_pass.apply(main_graph) add_quant_dequant_pass.apply(test_graph) scale_training_pass = OutScaleForTrainingPass(scope=scope, place=place) scale_training_pass.apply(main_graph) dev_name = '_gpu' if use_cuda else '_cpu' build_strategy = fluid.BuildStrategy() build_strategy.memory_optimize = False build_strategy.enable_inplace = False build_strategy.fuse_all_reduce_ops = False binary = fluid.CompiledProgram(main_graph.graph).with_data_parallel( loss_name=loss.name, build_strategy=build_strategy) iters = 5 batch_size = 8 train_reader = paddle.batch(paddle.reader.shuffle( paddle.dataset.mnist.train(), buf_size=500), batch_size=batch_size) feeder = fluid.DataFeeder(feed_list=feeds, place=place) with fluid.scope_guard(scope): for _ in range(iters): data = next(train_reader()) loss_v = exe.run(binary, feed=feeder.feed(data), fetch_list=[loss]) out_scale_infer_pass = OutScaleForInferencePass(scope=scope) out_scale_infer_pass.apply(test_graph) freeze_pass = QuantizationFreezePass( scope=scope, place=place, weight_bits=8, activation_bits=8, weight_quantize_type=weight_quant_type) mapping_table = load_dict() test_graph.out_node_mapping_table = mapping_table if act_quantize_func == None and weight_quantize_func == None: freeze_pass.apply(test_graph)
def quant_aware(program, place, config=None, scope=None, for_test=False, weight_quantize_func=None, act_quantize_func=None, weight_preprocess_func=None, act_preprocess_func=None, optimizer_func=None, executor=None, return_program=False): """Add quantization and dequantization operators to "program" for quantization training or testing. Args: program(paddle.static.Program): training or testing ``program``. place(paddle.CPUPlace or paddle.CUDAPlace): This parameter represents the executor run on which device. config(dict, optional): configs for quantization. if None, will use default config. Default: None. scope(paddle.static.Scope): Scope records the mapping between variable names and variables, similar to brackets in programming languages. Usually users can use `paddle.static.global_scope <https://www.paddlepaddle.org.cn/documentation/docs/zh/develop/api_cn/executor_cn/global_scope_cn.html>`_. When ``None`` will use `paddle.static.global_scope() <https://www.paddlepaddle.org.cn/documentation/docs/zh/develop/api_cn/executor_cn/global_scope_cn.html>`_ . Default: ``None``. for_test(bool): If the 'program' parameter is a test program, this parameter should be set to ``True``. Otherwise, set to ``False``.Default: False weight_quantize_func(function): Function that defines how to quantize weight. Using this can quickly test if user's quantization method works or not. In this function, user should both define quantization function and dequantization function, that is, the function's input is non-quantized weight and function returns dequantized weight. If None, will use quantization op defined by 'weight_quantize_type'. Default is None. act_quantize_func(function): Function that defines how to quantize activation. Using this can quickly test if user's quantization method works or not. In this function, user should both define quantization and dequantization process, that is, the function's input is non-quantized activation and function returns dequantized activation. If None, will use quantization op defined by 'activation_quantize_type'. Default is None. weight_preprocess_func(function): Function that defines how to preprocess weight before quantization. Using this can quickly test if user's preprocess method works or not. The function's input is non-quantized weight and function returns processed weight to be quantized. If None, the weight will be quantized directly. Default is None. act_preprocess_func(function): Function that defines how to preprocess activation before quantization. Using this can quickly test if user's preprocess method works or not. The function's input is non-quantized activation and function returns processed activation to be quantized. If None, the activation will be quantized directly. Default is None. optimizer_func(function): Fuction return a optimizer. When 'is_test' is False and user want to use self-defined quantization function and preprocess function, this function must be set. Default is None. exe(paddle.static.Executor): If user want to use self-defined quantization function and preprocess function, exe must be set for initialization. Default is None. return_program(bool): If user want return value is a Program rather than Compiled Program, This argument should be set True. Default is False. Returns: paddle.static.CompiledProgram | paddle.static.Program: Program with quantization and dequantization ``operators`` """ scope = paddle.static.global_scope() if not scope else scope if config is None: config = _quant_config_default else: assert isinstance(config, dict), "config must be dict" config = _parse_configs(config) _logger.info("quant_aware config {}".format(config)) main_graph = IrGraph(core.Graph(program.desc), for_test=for_test) transform_pass_ops = [] quant_dequant_ops = [] for op_type in config['quantize_op_types']: if op_type in TRANSFORM_PASS_OP_TYPES: transform_pass_ops.append(op_type) elif op_type in QUANT_DEQUANT_PASS_OP_TYPES: quant_dequant_ops.append(op_type) if len(transform_pass_ops) > 0: transform_pass = QuantizationTransformPass( scope=scope, place=place, weight_bits=config['weight_bits'], activation_bits=config['activation_bits'], activation_quantize_type=config['activation_quantize_type'], weight_quantize_type=config['weight_quantize_type'], window_size=config['window_size'], moving_rate=config['moving_rate'], quantizable_op_type=transform_pass_ops, skip_pattern=config['not_quant_pattern'], weight_quantize_func=weight_quantize_func, act_quantize_func=act_quantize_func, weight_preprocess_func=weight_preprocess_func, act_preprocess_func=act_preprocess_func, optimizer_func=optimizer_func, executor=executor) transform_pass.apply(main_graph) if len(quant_dequant_ops) > 0: quant_dequant_pass = AddQuantDequantPass( scope=scope, place=place, moving_rate=config['moving_rate'], quant_bits=config['activation_bits'], skip_pattern=config['not_quant_pattern'], quantizable_op_type=quant_dequant_ops) quant_dequant_pass.apply(main_graph) out_scale_training_pass = OutScaleForTrainingPass( scope=scope, place=place, moving_rate=config['moving_rate']) out_scale_training_pass.apply(main_graph) if (weight_preprocess_func is not None or act_preprocess_func is not None) and not for_test: _logger.info( "When a preprocess_func is used in quant_aware, Need to save a mapping table to match variable names in the convert phase." ) _logger.info( "The mapping table is saved as '{}'.".format(VARS_MAPPING_TABLE)) save_dict(main_graph.out_node_mapping_table) if for_test or return_program: quant_program = main_graph.to_program() else: quant_program = paddle.static.CompiledProgram(main_graph.graph) return quant_program
def main(): cfg = load_config(FLAGS.config) if 'architecture' in cfg: main_arch = cfg.architecture else: raise ValueError("'architecture' not specified in config file.") merge_config(FLAGS.opt) if 'log_iter' not in cfg: cfg.log_iter = 20 # check if set use_gpu=True in paddlepaddle cpu version check_gpu(cfg.use_gpu) if cfg.use_gpu: devices_num = fluid.core.get_cuda_device_count() else: devices_num = int( os.environ.get('CPU_NUM', multiprocessing.cpu_count())) if 'eval_feed' not in cfg: eval_feed = create(main_arch + 'EvalFeed') else: eval_feed = create(cfg.eval_feed) place = fluid.CUDAPlace(0) if cfg.use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) _, test_feed_vars = create_feed(eval_feed, False) eval_reader = create_reader(eval_feed, args_path=FLAGS.dataset_dir) #eval_pyreader.decorate_sample_list_generator(eval_reader, place) test_data_feed = fluid.DataFeeder(test_feed_vars.values(), place) assert os.path.exists(FLAGS.model_path) infer_prog, feed_names, fetch_targets = fluid.io.load_inference_model( dirname=FLAGS.model_path, executor=exe, model_filename='__model__.infer', params_filename='__params__') eval_keys = ['bbox', 'gt_box', 'gt_label', 'is_difficult'] eval_values = [ 'multiclass_nms_0.tmp_0', 'gt_box', 'gt_label', 'is_difficult' ] eval_cls = [] eval_values[0] = fetch_targets[0] results = eval_run(exe, infer_prog, eval_reader, eval_keys, eval_values, eval_cls, test_data_feed) resolution = None if 'mask' in results[0]: resolution = model.mask_head.resolution box_ap_stats = eval_results(results, eval_feed, cfg.metric, cfg.num_classes, resolution, False, FLAGS.output_eval) logger.info("freeze the graph for inference") test_graph = IrGraph(core.Graph(infer_prog.desc), for_test=True) freeze_pass = QuantizationFreezePass( scope=fluid.global_scope(), place=place, weight_quantize_type=FLAGS.weight_quant_type) freeze_pass.apply(test_graph) server_program = test_graph.to_program() fluid.io.save_inference_model(dirname=os.path.join(FLAGS.save_path, 'float'), feeded_var_names=feed_names, target_vars=fetch_targets, executor=exe, main_program=server_program, model_filename='model', params_filename='weights') logger.info("convert the weights into int8 type") convert_int8_pass = ConvertToInt8Pass(scope=fluid.global_scope(), place=place) convert_int8_pass.apply(test_graph) server_int8_program = test_graph.to_program() fluid.io.save_inference_model(dirname=os.path.join(FLAGS.save_path, 'int8'), feeded_var_names=feed_names, target_vars=fetch_targets, executor=exe, main_program=server_int8_program, model_filename='model', params_filename='weights')
def _predict(self, test_reader=None, model_path=None, batch_size=1, batch_num=1, skip_batch_num=0, transform_to_int8=False): place = fluid.CPUPlace() exe = fluid.Executor(place) inference_scope = fluid.executor.global_scope() with fluid.scope_guard(inference_scope): if os.path.exists(os.path.join(model_path, '__model__')): [inference_program, feed_target_names, fetch_targets ] = fluid.io.load_inference_model(model_path, exe) else: [inference_program, feed_target_names, fetch_targets ] = fluid.io.load_inference_model(model_path, exe, 'model', 'params') graph = IrGraph(core.Graph(inference_program.desc), for_test=True) if (self._debug): graph.draw('.', 'qat_orig', graph.all_op_nodes()) if (transform_to_int8): transform_to_mkldnn_int8_pass = Qat2Int8MkldnnPass( self._quantized_ops, _scope=inference_scope, _place=place, _core=core, _debug=self._debug) graph = transform_to_mkldnn_int8_pass.apply(graph) inference_program = graph.to_program() total_correct = 0 total_samples = 0 batch_times = [] ppses = [] # predictions per second iters = 0 infer_start_time = time.time() for data in test_reader(): if batch_num > 0 and iters >= batch_num: break if iters == skip_batch_num: total_samples = 0 infer_start_time = time.time() input0 = np.array([x[0] for x in data]).astype('int64') input1 = np.array([x[1] for x in data]).astype('int64') labels = np.array([x[2] for x in data]).astype('int64') start = time.time() out = exe.run(inference_program, feed={ feed_target_names[0]: input0, feed_target_names[1]: input1 }, fetch_list=fetch_targets) batch_time = (time.time() - start) * 1000 # in miliseconds batch_times.append(batch_time) batch_correct = self._get_batch_correct(out, labels) batch_len = len(data) total_samples += batch_len total_correct += batch_correct batch_acc = float(batch_correct) / float(batch_len) pps = batch_len / batch_time * 1000 ppses.append(pps) latency = batch_time / batch_len iters += 1 appx = ' (warm-up)' if iters <= skip_batch_num else '' _logger.info( 'batch {0}{4}, acc: {1:.4f}, latency: {2:.4f} ms, predictions per sec: {3:.2f}' .format(iters, batch_acc, latency, pps, appx)) # Postprocess benchmark data infer_total_time = time.time() - infer_start_time batch_latencies = batch_times[skip_batch_num:] batch_latency_avg = np.average(batch_latencies) latency_avg = batch_latency_avg / batch_size ppses = ppses[skip_batch_num:] pps_avg = np.average(ppses) acc_avg = float(np.sum(total_correct)) / float(total_samples) _logger.info( 'Total inference run time: {:.2f} s'.format(infer_total_time)) return acc_avg, pps_avg, latency_avg
def train(args): # parameters from arguments model_name = args.model pretrained_fp32_model = args.pretrained_fp32_model checkpoint = args.checkpoint model_save_dir = args.model_save_dir data_dir = args.data_dir activation_quant_type = args.act_quant_type weight_quant_type = args.wt_quant_type print("Using %s as the actiavtion quantize type." % activation_quant_type) print("Using %s as the weight quantize type." % weight_quant_type) startup_prog = fluid.Program() train_prog = fluid.Program() test_prog = fluid.Program() _, _, train_py_reader, train_cost, train_acc1, train_acc5, global_lr = build_program( is_train=True, main_prog=train_prog, startup_prog=startup_prog, args=args) image, out, test_py_reader, test_cost, test_acc1, test_acc5 = build_program( is_train=False, main_prog=test_prog, startup_prog=startup_prog, args=args) test_prog = test_prog.clone(for_test=True) place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup_prog) main_graph = IrGraph(core.Graph(train_prog.desc), for_test=False) test_graph = IrGraph(core.Graph(test_prog.desc), for_test=True) if pretrained_fp32_model: def if_exist(var): return os.path.exists(os.path.join(pretrained_fp32_model, var.name)) fluid.io.load_vars( exe, pretrained_fp32_model, main_program=train_prog, predicate=if_exist) if args.use_gpu: visible_device = os.getenv('CUDA_VISIBLE_DEVICES') if visible_device: device_num = len(visible_device.split(',')) else: device_num = subprocess.check_output( ['nvidia-smi', '-L']).decode().count('\n') else: device_num = 1 train_batch_size = args.batch_size / device_num test_batch_size = 1 if activation_quant_type == 'abs_max' else 8 train_reader = paddle.batch( reader.train(data_dir=data_dir), batch_size=train_batch_size, drop_last=True) test_reader = paddle.batch(reader.val(data_dir=data_dir), batch_size=test_batch_size) train_py_reader.decorate_paddle_reader(train_reader) test_py_reader.decorate_paddle_reader(test_reader) train_fetch_list = [train_cost.name, train_acc1.name, train_acc5.name, global_lr.name] test_fetch_list = [test_cost.name, test_acc1.name, test_acc5.name] # 1. Make some quantization transforms in the graph before training and testing. # According to the weight and activation quantization type, the graph will be added # some fake quantize operators and fake dequantize operators. transform_pass = QuantizationTransformPass( scope=fluid.global_scope(), place=place, activation_quantize_type=activation_quant_type, weight_quantize_type=weight_quant_type) transform_pass.apply(main_graph) transform_pass.apply(test_graph) if checkpoint: load_persistable_nodes(exe, checkpoint, main_graph) build_strategy = fluid.BuildStrategy() build_strategy.memory_optimize = False build_strategy.enable_inplace = False build_strategy.fuse_all_reduce_ops = False binary = fluid.CompiledProgram(main_graph.graph).with_data_parallel( loss_name=train_cost.name, build_strategy=build_strategy) test_prog = test_graph.to_program() params = models.__dict__[args.model]().params for pass_id in range(params["num_epochs"]): train_py_reader.start() train_info = [[], [], []] test_info = [[], [], []] train_time = [] batch_id = 0 try: while True: t1 = time.time() loss, acc1, acc5, lr = exe.run(binary, fetch_list=train_fetch_list) t2 = time.time() period = t2 - t1 loss = np.mean(np.array(loss)) acc1 = np.mean(np.array(acc1)) acc5 = np.mean(np.array(acc5)) train_info[0].append(loss) train_info[1].append(acc1) train_info[2].append(acc5) lr = np.mean(np.array(lr)) train_time.append(period) if batch_id % 10 == 0: print("Pass {0}, trainbatch {1}, loss {2}, \ acc1 {3}, acc5 {4}, lr {5}, time {6}" .format(pass_id, batch_id, loss, acc1, acc5, "%.6f" % lr, "%2.2f sec" % period)) sys.stdout.flush() batch_id += 1 except fluid.core.EOFException: train_py_reader.reset() train_loss = np.array(train_info[0]).mean() train_acc1 = np.array(train_info[1]).mean() train_acc5 = np.array(train_info[2]).mean() test_py_reader.start() test_batch_id = 0 try: while True: t1 = time.time() loss, acc1, acc5 = exe.run(program=test_prog, fetch_list=test_fetch_list) t2 = time.time() period = t2 - t1 loss = np.mean(loss) acc1 = np.mean(acc1) acc5 = np.mean(acc5) test_info[0].append(loss) test_info[1].append(acc1) test_info[2].append(acc5) if test_batch_id % 10 == 0: print("Pass {0},testbatch {1},loss {2}, \ acc1 {3},acc5 {4},time {5}" .format(pass_id, test_batch_id, loss, acc1, acc5, "%2.2f sec" % period)) sys.stdout.flush() test_batch_id += 1 except fluid.core.EOFException: test_py_reader.reset() test_loss = np.array(test_info[0]).mean() test_acc1 = np.array(test_info[1]).mean() test_acc5 = np.array(test_info[2]).mean() print("End pass {0}, train_loss {1}, train_acc1 {2}, train_acc5 {3}, " "test_loss {4}, test_acc1 {5}, test_acc5 {6}".format( pass_id, train_loss, train_acc1, train_acc5, test_loss, test_acc1, test_acc5)) sys.stdout.flush() save_checkpoint_path = os.path.join(model_save_dir, model_name, str(pass_id)) if not os.path.isdir(save_checkpoint_path): os.makedirs(save_checkpoint_path) save_persistable_nodes(exe, save_checkpoint_path, main_graph) model_path = os.path.join(model_save_dir, model_name, args.act_quant_type) float_path = os.path.join(model_path, 'float') int8_path = os.path.join(model_path, 'int8') mobile_path = os.path.join(model_path, 'mobile') if not os.path.isdir(model_path): os.makedirs(model_path) # 2. Freeze the graph after training by adjusting the quantize # operators' order for the inference. freeze_pass = QuantizationFreezePass( scope=fluid.global_scope(), place=place, weight_quantize_type=weight_quant_type) freeze_pass.apply(test_graph) server_program = test_graph.to_program() fluid.io.save_inference_model( dirname=float_path, feeded_var_names=[image.name], target_vars=[out], executor=exe, main_program=server_program) # 3. Convert the weights into int8_t type. # (This step is optional.) convert_int8_pass = ConvertToInt8Pass(scope=fluid.global_scope(), place=place) convert_int8_pass.apply(test_graph) server_int8_program = test_graph.to_program() fluid.io.save_inference_model( dirname=int8_path, feeded_var_names=[image.name], target_vars=[out], executor=exe, main_program=server_int8_program) # 4. Convert the freezed graph for paddle-mobile execution. # (This step is optional.) mobile_pass = TransformForMobilePass() mobile_pass.apply(test_graph) mobile_program = test_graph.to_program() fluid.io.save_inference_model( dirname=mobile_path, feeded_var_names=[image.name], target_vars=[out], executor=exe, main_program=mobile_program)