def init_predictor(args):
if args.model_dir is not "":
config = Config(args.model_dir)
else:
config = Config(args.model_file, args.params_file)
config.enable_memory_optim()
if args.tune:
config.collect_shape_range_info(shape_file)
if args.use_gpu:
config.enable_use_gpu(1000, 0)
if args.use_trt:
# using dynamic shpae mode, the max_batch_size will be ignored.
config.enable_tensorrt_engine(workspace_size=1 << 30,
max_batch_size=1,
min_subgraph_size=5,
precision_mode=PrecisionType.Float32,
use_static=False,
use_calib_mode=False)
if args.tuned_dynamic_shape:
config.enable_tuned_tensorrt_dynamic_shape(shape_file, True)
else:
# If not specific mkldnn, you can set the blas thread.
# The thread num should not be greater than the number of cores in the CPU.
config.set_cpu_math_library_num_threads(4)
config.enable_mkldnn()
predictor = create_predictor(config)
return predictor
def __init__(self, cfg, name='BMN'):
name = name.upper()
self.name = name
model_file = cfg[name]['model_file']
params_file = cfg[name]['params_file']
gpu_mem = cfg[name]['gpu_mem']
device_id = cfg[name]['device_id']
self.nms_thread = cfg[name]['nms_thread']
self.min_pred_score = cfg[name]['score_thread']
self.min_frame_thread = cfg['COMMON']['fps']
# model init
config = Config(model_file, params_file)
config.enable_use_gpu(gpu_mem, device_id)
config.switch_ir_optim(True) # default true
config.enable_memory_optim()
# use zero copy
config.switch_use_feed_fetch_ops(False)
self.predictor = create_predictor(config)
input_names = self.predictor.get_input_names()
self.input_tensor = self.predictor.get_input_handle(input_names[0])
output_names = self.predictor.get_output_names()
self.output1_tensor = self.predictor.get_output_handle(output_names[0])
self.output2_tensor = self.predictor.get_output_handle(output_names[1])
self.output3_tensor = self.predictor.get_output_handle(output_names[2])
def create_paddle_predictor(self, args, inference_model_dir=None):
if inference_model_dir is None:
inference_model_dir = args.inference_model_dir
params_file = os.path.join(inference_model_dir, "inference.pdiparams")
model_file = os.path.join(inference_model_dir, "inference.pdmodel")
config = Config(model_file, params_file)
if args.use_gpu:
config.enable_use_gpu(args.gpu_mem, 0)
else:
config.disable_gpu()
if args.enable_mkldnn:
# cache 10 different shapes for mkldnn to avoid memory leak
config.set_mkldnn_cache_capacity(10)
config.enable_mkldnn()
config.set_cpu_math_library_num_threads(args.cpu_num_threads)
if args.enable_profile:
config.enable_profile()
config.disable_glog_info()
config.switch_ir_optim(args.ir_optim) # default true
if args.use_tensorrt:
config.enable_tensorrt_engine(
precision_mode=Config.Precision.Half
if args.use_fp16 else Config.Precision.Float32,
max_batch_size=args.batch_size,
min_subgraph_size=30)
config.enable_memory_optim()
# use zero copy
config.switch_use_feed_fetch_ops(False)
predictor = create_predictor(config)
return predictor, config
def create_paddle_predictor(args):
config = Config(args.model_file, args.params_file)
if args.use_gpu:
config.enable_use_gpu(args.gpu_mem, 0)
else:
config.disable_gpu()
if args.enable_mkldnn:
# cache 10 different shapes for mkldnn to avoid memory leak
config.set_mkldnn_cache_capacity(10)
config.enable_mkldnn()
#config.disable_glog_info()
config.switch_ir_optim(args.ir_optim) # default true
if args.use_tensorrt:
config.enable_tensorrt_engine(
precision_mode=Config.Precision.Half
if args.use_fp16 else Config.Precision.Float32,
max_batch_size=args.batch_size)
config.enable_memory_optim()
# use zero copy
config.switch_use_feed_fetch_ops(False)
predictor = create_predictor(config)
return predictor
def __init__(self, cfg, name='ACTION'):
name = name.upper()
self.name = name
model_file = cfg[name]['model_file']
params_file = cfg[name]['params_file']
gpu_mem = cfg[name]['gpu_mem']
device_id = cfg[name]['device_id']
self.topk = cfg[name]['topk']
self.frame_offset = cfg[name]['nms_offset']
self.nms_thread = cfg[name]['nms_thread']
self.cls_thread = cfg[name]['classify_score_thread']
self.iou_thread = cfg[name]['iou_score_thread']
self.label_map_file = cfg['COMMON']['label_dic']
self.fps = cfg['COMMON']['fps']
self.nms_id = 5
# model init
config = Config(model_file, params_file)
config.enable_use_gpu(gpu_mem, device_id)
config.switch_ir_optim(True) # default true
config.enable_memory_optim()
# use zero copy
config.switch_use_feed_fetch_ops(False)
self.predictor = create_predictor(config)
input_names = self.predictor.get_input_names()
self.input1_tensor = self.predictor.get_input_handle(input_names[0])
#self.input2_tensor = self.predictor.get_input_handle(input_names[1])
output_names = self.predictor.get_output_names()
self.output1_tensor = self.predictor.get_output_handle(output_names[0])
self.output2_tensor = self.predictor.get_output_handle(output_names[1])
def __init__(self, cfg, name='PPTSM'):
name = name.upper()
self.name = name
model_file = cfg[name]['model_file']
params_file = cfg[name]['params_file']
gpu_mem = cfg[name]['gpu_mem']
device_id = cfg[name]['device_id']
# model init
config = Config(model_file, params_file)
config.enable_use_gpu(gpu_mem, device_id)
config.switch_ir_optim(True) # default true
config.enable_memory_optim()
# use zero copy
config.switch_use_feed_fetch_ops(False)
self.predictor = create_predictor(config)
input_names = self.predictor.get_input_names()
self.input_tensor = self.predictor.get_input_handle(input_names[0])
output_names = self.predictor.get_output_names()
print("output_names = ", output_names)
#self.output_tensor = self.predictor.get_output_handle(output_names[1])
self.output_tensor = self.predictor.get_output_handle(output_names[0])
def load_predictor(self, model_file, params_file):
config = Config(model_file, params_file)
if self.predictor_config["use_gpu"]:
config.enable_use_gpu(200, 0)
config.switch_ir_optim(True)
else:
config.disable_gpu()
config.set_cpu_math_library_num_threads(
self.predictor_config["cpu_threads"])
if self.predictor_config["enable_mkldnn"]:
try:
# cache 10 different shapes for mkldnn to avoid memory leak
config.set_mkldnn_cache_capacity(10)
config.enable_mkldnn()
except Exception as e:
logging.error(
"The current environment does not support `mkldnn`, so disable mkldnn."
)
config.disable_glog_info()
config.enable_memory_optim()
# use zero copy
config.switch_use_feed_fetch_ops(False)
predictor = create_predictor(config)
input_names = predictor.get_input_names()
output_names = predictor.get_output_names()
return predictor, input_names, output_names
def load_model(self,
model_dir,
use_gpu=False,
enable_mkldnn=False,
cpu_threads=1):
model = os.path.join(model_dir, '__model__')
params = os.path.join(model_dir, '__params__')
config = Config(model, params)
# 设置参数
if use_gpu:
config.enable_use_gpu(100, 0)
else:
config.disable_gpu()
config.set_cpu_math_library_num_threads(cpu_threads)
if enable_mkldnn:
config.enable_mkldnn()
config.set_mkldnn_cache_capacity(10)
config.disable_glog_info()
config.switch_ir_optim(True)
config.enable_memory_optim()
config.switch_use_feed_fetch_ops(False)
config.switch_specify_input_names(True)
# 通过参数加载模型预测器
predictor = create_predictor(config)
# 获取模型的输入输出
input_names = predictor.get_input_names()
output_names = predictor.get_output_names()
input_handle = predictor.get_input_handle(input_names[0])
output_handle = predictor.get_output_handle(output_names[0])
return predictor, input_handle, output_handle
def load_config(self, modelpath, use_gpu, gpu_id, use_mkldnn, cpu_threads):
'''
load the model config
modelpath: inference model path
use_gpu: use gpu or not
use_mkldnn: use mkldnn or not
'''
# 对运行位置进行配置
if use_gpu:
try:
int(os.environ.get('CUDA_VISIBLE_DEVICES'))
except Exception:
print(
'''Error! Unable to use GPU. Please set the environment variables "CUDA_VISIBLE_DEVICES=GPU_id" to use GPU. Now switch to CPU to continue...''')
use_gpu = False
if os.path.isdir(modelpath):
if os.path.exists(os.path.join(modelpath, "__params__")):
# __model__ + __params__
model = os.path.join(modelpath, "__model__")
params = os.path.join(modelpath, "__params__")
config = Config(model, params)
elif os.path.exists(os.path.join(modelpath, "params")):
# model + params
model = os.path.join(modelpath, "model")
params = os.path.join(modelpath, "params")
config = Config(model, params)
elif os.path.exists(os.path.join(modelpath, "__model__")):
# __model__ + others
config = Config(modelpath)
else:
raise Exception(
"Error! Can\'t find the model in: %s. Please check your model path." % os.path.abspath(modelpath))
elif os.path.exists(modelpath+".pdmodel"):
# *.pdmodel + *.pdiparams
model = modelpath+".pdmodel"
params = modelpath+".pdiparams"
config = Config(model, params)
else:
raise Exception(
"Error! Can\'t find the model in: %s. Please check your model path." % os.path.abspath(modelpath))
# 设置参数
if use_gpu:
config.enable_use_gpu(100, gpu_id)
else:
config.disable_gpu()
config.set_cpu_math_library_num_threads(cpu_threads)
if use_mkldnn:
config.enable_mkldnn()
config.disable_glog_info()
# 返回配置
return config
def load_predictor(model_dir,
run_mode='fluid',
batch_size=1,
use_gpu=False,
min_subgraph_size=3):
"""set AnalysisConfig, generate AnalysisPredictor
Args:
model_dir (str): root path of __model__ and __params__
use_gpu (bool): whether use gpu
Returns:
predictor (PaddlePredictor): AnalysisPredictor
Raises:
ValueError: predict by TensorRT need use_gpu == True.
"""
if not use_gpu and not run_mode == 'fluid':
raise ValueError(
"Predict by TensorRT mode: {}, expect use_gpu==True, but use_gpu == {}"
.format(run_mode, use_gpu))
if run_mode == 'trt_int8':
raise ValueError("TensorRT int8 mode is not supported now, "
"please use trt_fp32 or trt_fp16 instead.")
config = Config(os.path.join(model_dir, 'model.pdmodel'),
os.path.join(model_dir, 'model.pdiparams'))
precision_map = {
'trt_int8': Config.Precision.Int8,
'trt_fp32': Config.Precision.Float32,
'trt_fp16': Config.Precision.Half
}
if use_gpu:
# initial GPU memory(M), device ID
config.enable_use_gpu(200, 0)
# optimize graph and fuse op
# FIXME(dkp): ir optimize may prune variable inside graph
# and incur error in Paddle 2.0, e.g. in SSDLite
# FCOS model, set as False currently and should
# be set as True after switch_ir_optim fixed
config.switch_ir_optim(False)
else:
config.disable_gpu()
if run_mode in precision_map.keys():
config.enable_tensorrt_engine(workspace_size=1 << 10,
max_batch_size=batch_size,
min_subgraph_size=min_subgraph_size,
precision_mode=precision_map[run_mode],
use_static=False,
use_calib_mode=False)
# disable print log when predict
config.disable_glog_info()
# enable shared memory
config.enable_memory_optim()
# disable feed, fetch OP, needed by zero_copy_run
config.switch_use_feed_fetch_ops(False)
predictor = create_predictor(config)
return predictor
def init_predictor(args):
config = Config()
if args.model_dir == "":
config.set_model(args.model_file, args.params_file)
else:
config.set_model(args.model_dir)
#config.disable_glog_info()
config.enable_use_gpu(1000, 3)
predictor = create_predictor(config)
return predictor
def init_predictor(args):
if args.model_dir:
config = Config(args.model_dir)
else:
config = Config(args.model_file, args.params_file)
if args.use_gpu:
config.enable_use_gpu(1000, 0)
else:
config.disable_gpu()
print(config)
# config.delete('repeated_fc_relu_fuse_pass')
predictor = create_predictor(config)
return predictor
def init_predictor(args):
config = Config(os.path.join(args.model_dir, "inference.pdmodel"),
os.path.join(args.model_dir, "inference.pdiparams"))
config.enable_memory_optim()
if args.use_gpu:
config.enable_use_gpu(1000, 0)
else:
# If not specific mkldnn, you can set the blas thread.
# The thread num should not be greater than the number of cores in the CPU.
config.set_cpu_math_library_num_threads(4)
predictor = create_predictor(config)
return predictor
def auto_tune(args, dataset, img_nums):
"""
Use images to auto tune the dynamic shape for trt sub graph.
The tuned shape saved in args.auto_tuned_shape_file.
Args:
args(dict): input args.
dataset(dataset): an dataset.
img_nums(int): the nums of images used for auto tune.
Returns:
None
"""
logger.info("Auto tune the dynamic shape for GPU TRT.")
assert use_auto_tune(args)
num = min(len(dataset), img_nums)
cfg = DeployConfig(args.cfg)
pred_cfg = PredictConfig(cfg.model, cfg.params)
pred_cfg.enable_use_gpu(100, 0)
if not args.print_detail:
pred_cfg.disable_glog_info()
pred_cfg.collect_shape_range_info(args.auto_tuned_shape_file)
predictor = create_predictor(pred_cfg)
input_names = predictor.get_input_names()
input_handle = predictor.get_input_handle(input_names[0])
for idx, (img, _) in enumerate(dataset):
data = np.array([img])
input_handle.reshape(data.shape)
input_handle.copy_from_cpu(data)
try:
predictor.run()
except:
logger.info(
"Auto tune fail. Usually, the error is out of GPU memory, "
"because the model and image is too large. \n")
del predictor
if os.path.exists(args.auto_tuned_shape_file):
os.remove(args.auto_tuned_shape_file)
return
if idx + 1 >= num:
break
logger.info("Auto tune success.\n")
def init_predictor(args):
if args.model_dir is not "":
config = Config(args.model_dir)
else:
config = Config(args.model_file, args.params_file)
config.enable_memory_optim()
if args.use_gpu:
config.enable_use_gpu(1000, 0)
else:
# If not specific mkldnn, you can set the blas thread.
# The thread num should not be greater than the number of cores in the CPU.
config.set_cpu_math_library_num_threads(4)
predictor = create_predictor(config)
return predictor
def __init__(self,
model_file,
params_file,
use_mkldnn=True,
use_gpu=False,
device_id=0):
config = Config(model_file, params_file)
config.enable_memory_optim()
if use_gpu:
print("ENABLE_GPU")
config.enable_use_gpu(100, device_id)
if use_mkldnn:
config.enable_mkldnn()
self.predictor = create_predictor(config)
def auto_tune(args, imgs, img_nums):
"""
Use images to auto tune the dynamic shape for trt sub graph.
The tuned shape saved in args.auto_tuned_shape_file.
Args:
args(dict): input args.
imgs(str, list[str]): the path for images.
img_nums(int): the nums of images used for auto tune.
Returns:
None
"""
logger.info("Auto tune the dynamic shape for GPU TRT.")
assert use_auto_tune(args)
if not isinstance(imgs, (list, tuple)):
imgs = [imgs]
num = min(len(imgs), img_nums)
cfg = DeployConfig(args.cfg)
pred_cfg = PredictConfig(cfg.model, cfg.params)
pred_cfg.enable_use_gpu(100, 0)
if not args.print_detail:
pred_cfg.disable_glog_info()
pred_cfg.collect_shape_range_info(args.auto_tuned_shape_file)
predictor = create_predictor(pred_cfg)
input_names = predictor.get_input_names()
input_handle = predictor.get_input_handle(input_names[0])
for i in range(0, num):
data = np.array([cfg.transforms(imgs[i])[0]])
input_handle.reshape(data.shape)
input_handle.copy_from_cpu(data)
try:
predictor.run()
except:
logger.info(
"Auto tune fail. Usually, the error is out of GPU memory, "
"because the model and image is too large. \n")
del predictor
if os.path.exists(args.auto_tuned_shape_file):
os.remove(args.auto_tuned_shape_file)
return
logger.info("Auto tune success.\n")
def init_predictor(args):
if args.model_dir is not "":
config = Config(args.model_dir)
else:
config = Config(args.model_file, args.params_file)
config.enable_memory_optim()
config.enable_use_gpu(100, 0)
# using dynamic shpae mode, the max_batch_size will be ignored.
config.enable_tensorrt_engine(workspace_size=1 << 30,
max_batch_size=1,
min_subgraph_size=5,
precision_mode=PrecisionType.Float32,
use_static=False,
use_calib_mode=False)
head_number = 12
names = [
"placeholder_0", "placeholder_1", "placeholder_2", "stack_0.tmp_0"
]
min_input_shape = [1, 1, 1]
max_input_shape = [100, 128, 1]
opt_input_shape = [10, 60, 1]
config.set_trt_dynamic_shape_info(
{
names[0]: min_input_shape,
names[1]: min_input_shape,
names[2]: min_input_shape,
names[3]: [1, head_number, 1, 1]
}, {
names[0]: max_input_shape,
names[1]: max_input_shape,
names[2]: max_input_shape,
names[3]: [100, head_number, 128, 128]
}, {
names[0]: opt_input_shape,
names[1]: opt_input_shape,
names[2]: opt_input_shape,
names[3]: [10, head_number, 60, 60]
})
predictor = create_predictor(config)
return predictor
def __init__(self, args):
self.cfg = DeployConfig(args.cfg)
self.args = args
pred_cfg = PredictConfig(self.cfg.model, self.cfg.params)
pred_cfg.disable_glog_info()
if self.args.use_gpu:
pred_cfg.enable_use_gpu(100, 0)
if self.args.use_trt:
ptype = PrecisionType.Int8 if args.use_int8 else PrecisionType.Float32
pred_cfg.enable_tensorrt_engine(workspace_size=1 << 30,
max_batch_size=1,
min_subgraph_size=3,
precision_mode=ptype,
use_static=False,
use_calib_mode=False)
self.predictor = create_predictor(pred_cfg)
def collect_dynamic_shape(args):
if not is_support_collecting():
logger.error("The Paddle does not support collecting dynamic shape, " \
"please reinstall the PaddlePaddle (latest gpu version).")
# prepare config
cfg = DeployConfig(args.config)
pred_cfg = PredictConfig(cfg.model, cfg.params)
pred_cfg.enable_use_gpu(1000, 0)
pred_cfg.collect_shape_range_info(args.dynamic_shape_path)
# create predictor
predictor = create_predictor(pred_cfg)
input_names = predictor.get_input_names()
input_handle = predictor.get_input_handle(input_names[0])
# get images
img_path_list, _ = get_image_list(args.image_path)
if not isinstance(img_path_list, (list, tuple)):
img_path_list = [img_path_list]
logger.info(f"The num of images is {len(img_path_list)} \n")
# collect
progbar_val = progbar.Progbar(target=len(img_path_list))
for idx, img_path in enumerate(img_path_list):
data = np.array([cfg.transforms(img_path)[0]])
input_handle.reshape(data.shape)
input_handle.copy_from_cpu(data)
try:
predictor.run()
except:
logger.info(
"Fail to collect dynamic shape. Usually, the error is out of "
"GPU memory, for the model and image are too large.\n")
del predictor
if os.path.exists(args.dynamic_shape_path):
os.remove(args.dynamic_shape_path)
progbar_val.update(idx + 1)
logger.info(f"The dynamic shape is save in {args.dynamic_shape_path}")
def get_config(self, model, params, tuned=False):
config = Config()
config.set_model_buffer(model, len(model), params, len(params))
config.enable_use_gpu(100, 0)
config.set_optim_cache_dir('tuned_test')
if tuned:
config.collect_shape_range_info('shape_range.pbtxt')
else:
config.enable_tensorrt_engine(
workspace_size=1024,
max_batch_size=1,
min_subgraph_size=0,
precision_mode=paddle.inference.PrecisionType.Float32,
use_static=True,
use_calib_mode=False)
config.enable_tuned_tensorrt_dynamic_shape('shape_range.pbtxt',
True)
return config
def __init__(self, args):
self.cfg = DeployConfig(args.cfg)
self.args = args
self.compose = T.Compose(self.cfg.transforms)
resize_h, resize_w = args.input_shape
self.disflow = cv2.DISOpticalFlow_create(
cv2.DISOPTICAL_FLOW_PRESET_ULTRAFAST)
self.prev_gray = np.zeros((resize_h, resize_w), np.uint8)
self.prev_cfd = np.zeros((resize_h, resize_w), np.float32)
self.is_init = True
pred_cfg = PredictConfig(self.cfg.model, self.cfg.params)
pred_cfg.disable_glog_info()
if self.args.use_gpu:
pred_cfg.enable_use_gpu(100, 0)
self.predictor = create_predictor(pred_cfg)
if self.args.test_speed:
self.cost_averager = TimeAverager()
def init_predictor(self, cfg):
model_dir = cfg['model_dir']
params_file = cfg['params_file']
use_gpu = cfg['use_gpu']
config = Config(model_dir, params_file)
# config.enable_memory_optim()
if use_gpu:
config.enable_use_gpu(1000, 0)
else:
# If not specific mkldnn, you can set the blas thread.
# The thread num should not be greater than the number of cores in the CPU.
config.set_cpu_math_library_num_threads(4)
config.enable_mkldnn()
config.disable_glog_info()
predictor = create_predictor(config)
return predictor
def init_predictor(args):
if args.model_dir:
config = Config(args.model_dir)
else:
config = Config(args.model_file, args.params_file)
if args.use_gpu:
config.enable_use_gpu(1000, 0)
if args.enable_tensorRT:
config.enable_tensorrt_engine(
max_batch_size=args.batchsize,
min_subgraph_size=1,
precision_mode=paddle.inference.PrecisionType.Float32)
else:
config.disable_gpu()
config.delete_pass("repeated_fc_relu_fuse_pass")
config.set_cpu_math_library_num_threads(args.cpu_threads)
if args.enable_mkldnn:
config.enable_mkldnn()
predictor = create_predictor(config)
return predictor, config
def _set_config(self):
"""
predictor config setting.
"""
# create default cpu predictor
cpu_config = Config(self.default_pretrained_model_path)
cpu_config.disable_glog_info()
cpu_config.disable_gpu()
self.cpu_predictor = create_predictor(cpu_config)
# create predictors using various types of devices
# npu
npu_id = self._get_device_id("FLAGS_selected_npus")
if npu_id != -1:
# use npu
npu_config = Config(self.default_pretrained_model_path)
npu_config.disable_glog_info()
npu_config.enable_npu(device_id=npu_id)
self.npu_predictor = create_predictor(npu_config)
# gpu
gpu_id = self._get_device_id("CUDA_VISIBLE_DEVICES")
if gpu_id != -1:
# use gpu
gpu_config = Config(self.default_pretrained_model_path)
gpu_config.disable_glog_info()
gpu_config.enable_use_gpu(memory_pool_init_size_mb=1000,
device_id=gpu_id)
self.gpu_predictor = create_predictor(gpu_config)
# xpu
xpu_id = self._get_device_id("XPU_VISIBLE_DEVICES")
if xpu_id != -1:
# use xpu
xpu_config = Config(self.default_pretrained_model_path)
xpu_config.disable_glog_info()
xpu_config.enable_xpu(100)
self.xpu_predictor = create_predictor(xpu_config)
def load_config(self, modelpath, use_gpu, use_mkldnn, combined):
'''
load the model config
modelpath: inference model path
use_gpu: use gpu or not
use_mkldnn: use mkldnn or not
combined: inference model format is combined or not
'''
# 对运行位置进行配置
if use_gpu:
try:
int(os.environ.get('CUDA_VISIBLE_DEVICES'))
except Exception:
print(
'Error! Unable to use GPU. Please set the environment variables "CUDA_VISIBLE_DEVICES=GPU_id" to use GPU.'
)
use_gpu = False
# 加载模型参数
if combined:
model = os.path.join(modelpath, "__model__")
params = os.path.join(modelpath, "__params__")
config = Config(model, params)
else:
config = Config(modelpath)
# 设置参数
if use_gpu:
config.enable_use_gpu(100, 0)
else:
config.disable_gpu()
if use_mkldnn:
config.enable_mkldnn()
# 返回配置
return config
def create_paddle_predictor(args):
config = Config(args.model_file, args.params_file)
if args.use_gpu:
config.enable_use_gpu(args.gpu_mem, 0)
else:
config.disable_gpu()
if args.use_mkldnn:
config.enable_mkldnn()
config.set_cpu_math_library_num_threads(args.cpu_num_threads)
config.set_mkldnn_cache_capacity(100)
config.disable_glog_info()
config.switch_ir_optim(args.ir_optim) # default true
config.enable_memory_optim()
# use zero copy
config.switch_use_feed_fetch_ops(False)
predictor = create_predictor(config)
return predictor
def init_model(self):
from paddle.inference import Config
from paddle.inference import PrecisionType
from paddle.inference import create_predictor
precision_mode = PrecisionType.Float32
use_calib_mode = False
if self.param_type == "fp16":
precision_mode = PrecisionType.Half
elif self.param_type == "int8":
precision_mode = PrecisionType.Int8
use_calib_mode = True
mode_path = os.path.join(self.model_dir,"yolov3/__model__")
param_path = os.path.join(self.model_dir,"yolov3/__params__")
config = Config(mode_path, param_path)
config.enable_use_gpu(100, 0)
config.switch_ir_optim(True)
size = (self.batch_size * 101) << 20
config.enable_tensorrt_engine(
workspace_size= size,
max_batch_size=self.batch_size,
min_subgraph_size=3,
precision_mode=precision_mode,
use_static=False,
use_calib_mode=use_calib_mode)
if not self.debug:
config.disable_glog_info()
else:
config.enable_profile()
config.enable_memory_optim()
config.switch_use_feed_fetch_ops(False)
config.enable_mkldnn()
#exit(1)
self.predictor = create_predictor(config)
def init_predictor(args):
if args.model_dir:
has_model = 0
pdmodel_name = 0
pdiparams_name = 0
for file_name in os.listdir(args.model_dir):
if re.search("__model__", file_name):
has_model = 1
if file_name.endswith(".pdmodel"):
pdmodel_name = os.path.join(args.model_dir, file_name)
if file_name.endswith(".pdiparams"):
pdiparams_name = os.path.join(args.model_dir, file_name)
if has_model == 1:
config = Config(args.model_dir)
elif pdmodel_name and pdiparams_name:
config = Config(pdmodel_name, pdiparams_name)
else:
raise ValueError(
"config setting error, please check your model path")
else:
config = Config(args.model_file, args.params_file)
if args.use_gpu:
config.enable_use_gpu(1000, 0)
if args.enable_tensorRT:
config.enable_tensorrt_engine(
max_batch_size=args.batchsize,
min_subgraph_size=9,
precision_mode=paddle.inference.PrecisionType.Float32)
else:
config.disable_gpu()
config.set_cpu_math_library_num_threads(args.cpu_threads)
if args.enable_mkldnn:
config.enable_mkldnn()
config.delete_pass("scale_matmul_fuse_pass")
predictor = create_predictor(config)
return predictor, config
def load_model(self, modelpath, use_gpu, use_mkldnn, combined):
# 对运行位置进行配置
if use_gpu:
try:
int(os.environ.get('CUDA_VISIBLE_DEVICES'))
except Exception:
print(
'Error! Unable to use GPU. Please set the environment variables "CUDA_VISIBLE_DEVICES=GPU_id" to use GPU.'
)
use_gpu = False
# 加载模型参数
if combined:
model = os.path.join(modelpath, "__model__")
params = os.path.join(modelpath, "__params__")
config = Config(model, params)
else:
config = Config(modelpath)
# 设置参数
if use_gpu:
config.enable_use_gpu(100, 0)
else:
config.disable_gpu()
if use_mkldnn:
config.enable_mkldnn()
config.disable_glog_info()
config.switch_ir_optim(True)
config.enable_memory_optim()
config.switch_use_feed_fetch_ops(False)
config.switch_specify_input_names(True)
# 通过参数加载模型预测器
predictor = create_predictor(config)
# 返回预测器
return predictor
