def load_model(self, modelpath, use_gpu):
# 对运行位置进行配置
if use_gpu:
try:
places = os.environ["CUDA_VISIBLE_DEVICES"]
places = int(places[0])
except Exception as e:
print(
'Error: %s. Please set the environment variables "CUDA_VISIBLE_DEVICES".'
% e)
use_gpu = False
# 加载模型参数
config = AnalysisConfig(modelpath)
# 设置参数
if use_gpu:
config.enable_use_gpu(100, places)
else:
config.disable_gpu()
config.enable_mkldnn()
config.disable_glog_info()
config.switch_ir_optim(True)
config.enable_memory_optim()
config.switch_use_feed_fetch_ops(False)
config.switch_specify_input_names(True)
# 通过参数加载模型预测器
predictor = create_paddle_predictor(config)
# 返回预测器
return predictor
def create_predictor(mode):
"""
create predictor for inference
:param args: params for prediction engine
:param mode: mode
:return: predictor
"""
if mode == "det":
model_dir = "./src/ai/ocr_paddle/inference/ch_ppocr_mobile_v1.1_det_infer"
elif mode == 'cls':
model_dir = "./src/ai/ocr_paddle/inference/ch_ppocr_mobile_v1.1_cls_infer"
elif mode == 'rec':
model_dir = "./src/ai/ocr_paddle/inference/ch_ppocr_mobile_v1.1_rec_infer"
else:
raise ValueError(
"'mode' of create_predictor() can only be one of ['det', 'cls', 'rec']"
)
if model_dir is None:
logger.info("not find {} model file path {}".format(mode, model_dir))
sys.exit(0)
model_file_path = model_dir + "/model"
params_file_path = model_dir + "/params"
if not os.path.exists(model_file_path):
logger.info("not find model file path {}".format(model_file_path))
sys.exit(0)
if not os.path.exists(params_file_path):
logger.info("not find params file path {}".format(params_file_path))
sys.exit(0)
config = AnalysisConfig(model_file_path, params_file_path)
config.disable_gpu()
config.set_cpu_math_library_num_threads(6)
enable_mkldnn = False
if enable_mkldnn:
# cache 10 different shapes for mkldnn to avoid memory leak
config.set_mkldnn_cache_capacity(10)
config.enable_mkldnn()
# config.enable_memory_optim()
config.disable_glog_info()
use_zero_copy_run = False
if use_zero_copy_run:
config.delete_pass("conv_transpose_eltwiseadd_bn_fuse_pass")
config.switch_use_feed_fetch_ops(False)
else:
config.switch_use_feed_fetch_ops(True)
predictor = create_paddle_predictor(config)
input_names = predictor.get_input_names()
for name in input_names:
input_tensor = predictor.get_input_tensor(name)
output_names = predictor.get_output_names()
output_tensors = []
for output_name in output_names:
output_tensor = predictor.get_output_tensor(output_name)
output_tensors.append(output_tensor)
return predictor, input_tensor, output_tensors
def _get_analysis_config(self,
use_gpu=False,
use_trt=False,
use_mkldnn=False):
'''
Return a new object of AnalysisConfig.
'''
config = AnalysisConfig(os.path.join(self.path, "model"),
os.path.join(self.path, "params"))
config.disable_gpu()
config.switch_specify_input_names(True)
config.switch_ir_optim(True)
config.switch_use_feed_fetch_ops(False)
if use_gpu:
config.enable_use_gpu(100, 0)
if use_trt:
config.enable_tensorrt_engine(
self.trt_parameters.workspace_size,
self.trt_parameters.max_batch_size,
self.trt_parameters.min_subgraph_size,
self.trt_parameters.precision,
self.trt_parameters.use_static,
self.trt_parameters.use_calib_mode)
elif use_mkldnn:
config.enable_mkldnn()
return config
def create_predictor(args, mode, model_path):
model_dir = model_path
model_file_path = model_dir + "/model"
params_file_path = model_dir + "/params"
assert os.path.exists(model_file_path)
assert os.path.exists(params_file_path)
config = AnalysisConfig(model_file_path, params_file_path)
# use CPU
config.disable_gpu()
config.set_cpu_math_library_num_threads(6)
if args['enable_mkldnn']:
config.enable_mkldnn()
# config.enable_memory_optim()
config.disable_glog_info()
if args['use_zero_copy_run']:
config.delete_pass("conv_transpose_eltwiseadd_bn_fuse_pass")
config.switch_use_feed_fetch_ops(False)
else:
config.switch_use_feed_fetch_ops(True)
predictor = create_paddle_predictor(config)
input_names = predictor.get_input_names()
for name in input_names:
input_tensor = predictor.get_input_tensor(name)
output_names = predictor.get_output_names()
output_tensors = []
for output_name in output_names:
output_tensor = predictor.get_output_tensor(output_name)
output_tensors.append(output_tensor)
return predictor, input_tensor, output_tensors
def _set_config(self):
"""
predictor config setting
"""
model_file_path = os.path.join(self.pretrained_model_path, 'model')
params_file_path = os.path.join(self.pretrained_model_path, 'params')
config = AnalysisConfig(model_file_path, params_file_path)
try:
_places = os.environ["CUDA_VISIBLE_DEVICES"]
int(_places[0])
use_gpu = True
except:
use_gpu = False
if use_gpu:
config.enable_use_gpu(8000, 0)
else:
config.disable_gpu()
if self.enable_mkldnn:
config.enable_mkldnn()
config.disable_glog_info()
# use zero copy
config.delete_pass("conv_transpose_eltwiseadd_bn_fuse_pass")
config.switch_use_feed_fetch_ops(False)
self.predictor = create_paddle_predictor(config)
input_names = self.predictor.get_input_names()
self.input_tensor = self.predictor.get_input_tensor(input_names[0])
output_names = self.predictor.get_output_names()
self.output_tensors = []
for output_name in output_names:
output_tensor = self.predictor.get_output_tensor(output_name)
self.output_tensors.append(output_tensor)
def create_predictor(args, mode, logger):
if mode == "det":
model_dir = args.det_model_dir
elif mode == 'cls':
model_dir = args.cls_model_dir
else:
model_dir = args.rec_model_dir
if model_dir is None:
logger.info("not find {} model file path {}".format(mode, model_dir))
sys.exit(0)
model_file_path = model_dir + "/inference.pdmodel"
params_file_path = model_dir + "/inference.pdiparams"
if not os.path.exists(model_file_path):
logger.info("not find model file path {}".format(model_file_path))
sys.exit(0)
if not os.path.exists(params_file_path):
logger.info("not find params file path {}".format(params_file_path))
sys.exit(0)
config = AnalysisConfig(model_file_path, params_file_path)
if args.use_gpu:
config.enable_use_gpu(args.gpu_mem, 0)
if args.use_tensorrt:
config.enable_tensorrt_engine(
precision_mode=AnalysisConfig.Precision.Half
if args.use_fp16 else AnalysisConfig.Precision.Float32,
max_batch_size=args.max_batch_size)
else:
config.disable_gpu()
config.set_cpu_math_library_num_threads(6)
if args.enable_mkldnn:
# cache 10 different shapes for mkldnn to avoid memory leak
config.set_mkldnn_cache_capacity(10)
config.enable_mkldnn()
# config.enable_memory_optim()
config.disable_glog_info()
if args.use_zero_copy_run:
config.delete_pass("conv_transpose_eltwiseadd_bn_fuse_pass")
config.switch_use_feed_fetch_ops(False)
else:
config.switch_use_feed_fetch_ops(True)
predictor = create_paddle_predictor(config)
input_names = predictor.get_input_names()
for name in input_names:
input_tensor = predictor.get_input_tensor(name)
output_names = predictor.get_output_names()
output_tensors = []
for output_name in output_names:
output_tensor = predictor.get_output_tensor(output_name)
output_tensors.append(output_tensor)
return predictor, input_tensor, output_tensors
def __init__(self):
"""
create predictor manager
"""
self.get_predictor_timeout = float(
config.get('get.predictor.timeout', default_value=0.5))
predictor_count = 0
enable_mkl = False
gpu_memory = 200
gpu_device_ids = []
model_dir = config.get('model.dir')
device_type = config.get('device.type')
if device_type == PredictorManager.CPU_DEVICE:
cpu_predictor_count = int(
config.getint('cpu.predictor.count', default_value=0))
predictor_count = cpu_predictor_count
enable_mkl = config.getboolean('cpu.enable_mkl',
default_value=False)
elif device_type == PredictorManager.GPU_DEVICE:
gpu_predictor_count = int(
config.getint('gpu.predictor.count', default_value=0))
predictor_count = gpu_predictor_count
gpu_memory = config.getint('gpu.predictor.memory',
default_value=200)
gpu_device_ids = config.get('gpu.predictor.device.id').split(',')
gpu_device_ids = map(int, gpu_device_ids)
if PYTHON_VERSION == 3:
gpu_device_ids = list(gpu_device_ids)
assert len(
gpu_device_ids
) == gpu_predictor_count, "gpu predictor count doesn't match device count"
else:
raise Exception("no device to run predictor!")
assert predictor_count > 0, "no device to predict"
logger.info(
"device type:{} predictor count:{} model dir:{} get predictor timeout:{}s"
.format(device_type, predictor_count, model_dir,
self.get_predictor_timeout))
self.predictor_queue = Queue(maxsize=predictor_count)
for i in range(predictor_count):
# Set config
predictor_config = AnalysisConfig(model_dir)
# predictor_config.specify_input_name()
if device_type == PredictorManager.CPU_DEVICE:
predictor_config.disable_gpu()
if enable_mkl:
predictor_config.enable_mkldnn()
else:
device_id = gpu_device_ids[i]
predictor_config.enable_use_gpu(gpu_memory, device_id)
# Create PaddlePredictor
predictor = create_paddle_predictor(predictor_config)
self.predictor_queue.put(predictor)
def set_config(self, model_flie, params_file, use_feed_fetch_ops,
specify_input_names):
config = AnalysisConfig(model_flie, params_file)
config.disable_gpu()
config.enable_mkldnn()
config.disable_glog_info()
config.switch_ir_optim(True)
config.switch_use_feed_fetch_ops(use_feed_fetch_ops)
config.switch_specify_input_names(specify_input_names)
return config
def __init__(self, model_file, params_file, use_mkldnn=True, use_gpu = False, device_id = 0):
config = AnalysisConfig(model_file, params_file)
config.switch_use_feed_fetch_ops(False)
config.switch_specify_input_names(True)
config.enable_memory_optim()
if use_gpu:
print ("ENABLE_GPU")
config.enable_use_gpu(100, device_id)
if use_mkldnn:
config.enable_mkldnn()
self.predictor = create_paddle_predictor(config)
def __load_inference_model(self, model_path, use_gpu):
"""
:param meta_path:
:return:
"""
check_cuda(use_gpu)
config = AnalysisConfig(model_path + "/" + "model", model_path + "/" + "params")
if use_gpu:
config.enable_use_gpu(1024)
else:
config.disable_gpu()
config.enable_mkldnn()
inference = create_paddle_predictor(config.to_native_config())
return inference
def set_config(model_path):
config = None
if os.path.exists(os.path.join(model_path, '__model__')):
config = AnalysisConfig(model_path)
else:
config = AnalysisConfig(model_path + '/model', model_path + '/params')
if test_args.use_analysis:
config.switch_ir_optim(True)
config.enable_mkldnn()
config.set_mkldnn_cache_capacity(test_args.mkldnn_cache_capacity)
config.set_cpu_math_library_num_threads(test_args.num_threads)
else:
config.to_native_config()
return config
def create_predictor(args, mode):
if mode == "det":
model_dir = args.det_model_dir
else:
model_dir = args.rec_model_dir
if model_dir is None:
logger.info("not find {} model file path {}".format(mode, model_dir))
sys.exit(0)
model_file_path = model_dir + "/model"
params_file_path = model_dir + "/params"
if not os.path.exists(model_file_path):
logger.info("not find model file path {}".format(model_file_path))
sys.exit(0)
if not os.path.exists(params_file_path):
logger.info("not find params file path {}".format(params_file_path))
sys.exit(0)
config = AnalysisConfig(model_file_path, params_file_path)
if args.use_gpu:
config.enable_use_gpu(args.gpu_mem, 0)
else:
config.disable_gpu()
config.set_cpu_math_library_num_threads(6)
if args.enable_mkldnn:
config.enable_mkldnn()
#config.enable_memory_optim()
config.disable_glog_info()
if args.use_zero_copy_run:
config.delete_pass("conv_transpose_eltwiseadd_bn_fuse_pass")
config.switch_use_feed_fetch_ops(False)
else:
config.switch_use_feed_fetch_ops(True)
predictor = create_paddle_predictor(config)
input_names = predictor.get_input_names()
for name in input_names:
input_tensor = predictor.get_input_tensor(name)
output_names = predictor.get_output_names()
output_tensors = []
for output_name in output_names:
output_tensor = predictor.get_output_tensor(output_name)
output_tensors.append(output_tensor)
return predictor, input_tensor, output_tensors
def _get_analysis_config(self,
use_gpu=False,
use_trt=False,
use_mkldnn=False):
'''
Return a new object of AnalysisConfig.
'''
config = AnalysisConfig(self.path)
config.disable_gpu()
config.switch_specify_input_names(True)
config.switch_ir_optim(True)
config.switch_use_feed_fetch_ops(False)
if use_gpu:
config.enable_use_gpu(100, 0)
if use_trt:
config.enable_tensorrt_engine(
self.trt_parameters.workspace_size,
self.trt_parameters.max_batch_size,
self.trt_parameters.min_subgraph_size,
self.trt_parameters.precision,
self.trt_parameters.use_static,
self.trt_parameters.use_calib_mode)
if self.trt_parameters.use_inspector:
config.enable_tensorrt_inspector()
self.assertTrue(
config.tensorrt_inspector_enabled(),
"The inspector option is not set correctly.")
if self.dynamic_shape_params:
config.set_trt_dynamic_shape_info(
self.dynamic_shape_params.min_input_shape,
self.dynamic_shape_params.max_input_shape,
self.dynamic_shape_params.optim_input_shape,
self.dynamic_shape_params.disable_trt_plugin_fp16)
if self.enable_tensorrt_varseqlen:
config.enable_tensorrt_varseqlen()
elif use_mkldnn:
config.enable_mkldnn()
if self.enable_mkldnn_bfloat16:
config.enable_mkldnn_bfloat16()
print('config summary:', config.summary())
return config
def _set_config(self, pretrained_model_path):
"""
predictor config path
"""
model_file_path = os.path.join(pretrained_model_path, 'model')
params_file_path = os.path.join(pretrained_model_path, 'params')
config = AnalysisConfig(model_file_path, params_file_path)
try:
_places = os.environ["CUDA_VISIBLE_DEVICES"]
int(_places[0])
use_gpu = True
except:
use_gpu = False
if use_gpu:
config.enable_use_gpu(8000, 0)
else:
config.disable_gpu()
if self.enable_mkldnn:
# cache 10 different shapes for mkldnn to avoid memory leak
config.set_mkldnn_cache_capacity(10)
config.enable_mkldnn()
config.disable_glog_info()
config.delete_pass("conv_transpose_eltwiseadd_bn_fuse_pass")
config.switch_use_feed_fetch_ops(False)
predictor = create_paddle_predictor(config)
input_names = predictor.get_input_names()
input_tensor = predictor.get_input_tensor(input_names[0])
output_names = predictor.get_output_names()
output_tensors = []
for output_name in output_names:
output_tensor = predictor.get_output_tensor(output_name)
output_tensors.append(output_tensor)
return predictor, input_tensor, output_tensors
def set_config(self,
model_path,
num_threads,
mkldnn_cache_capacity,
warmup_data=None,
use_analysis=False,
enable_ptq=False):
config = AnalysisConfig(model_path)
config.set_cpu_math_library_num_threads(num_threads)
if use_analysis:
config.disable_gpu()
config.switch_use_feed_fetch_ops(True)
config.switch_ir_optim(True)
config.enable_mkldnn()
config.set_mkldnn_cache_capacity(mkldnn_cache_capacity)
if enable_ptq:
# This pass to work properly, must be added before fc_fuse_pass
config.pass_builder().insert_pass(5, "fc_lstm_fuse_pass")
config.enable_quantizer()
config.quantizer_config().set_quant_data(warmup_data)
config.quantizer_config().set_quant_batch_size(1)
return config
def set_config_ptq(model_path, warmup_data):
config = None
if os.path.exists(os.path.join(model_path, '__model__')):
config = AnalysisConfig(model_path)
else:
config = AnalysisConfig(model_path + '/model', model_path + '/params')
config.switch_ir_optim(True)
# This pass must be added before fc_fuse_pass to work properly
config.pass_builder().insert_pass(5, "fc_lstm_fuse_pass")
config.pass_builder().append_pass("fc_mkldnn_pass")
config.enable_mkldnn()
config.set_mkldnn_cache_capacity(test_args.mkldnn_cache_capacity)
config.set_cpu_math_library_num_threads(test_args.num_threads)
config.enable_quantizer()
config.quantizer_config().set_quant_data(warmup_data)
config.quantizer_config().set_quant_batch_size(1)
ops_to_quantize = set()
if len(test_args.ops_to_quantize) > 0:
ops_to_quantize = set(test_args.ops_to_quantize.split(','))
config.quantizer_config().set_enabled_op_types(ops_to_quantize)
return config