def test_quantization_saved(self):
from lpot.utils.pytorch import load
model = copy.deepcopy(self.model)
for fake_yaml in ['qat_yaml.yaml', 'ptq_yaml.yaml']:
if fake_yaml == 'ptq_yaml.yaml':
model.eval().fuse_model()
quantizer = Quantization(fake_yaml)
dataset = quantizer.dataset('dummy', (100, 3, 256, 256),
label=True)
quantizer.model = common.Model(model)
quantizer.calib_dataloader = common.DataLoader(dataset)
quantizer.eval_dataloader = common.DataLoader(dataset)
if fake_yaml == 'qat_yaml.yaml':
quantizer.q_func = q_func
q_model = quantizer()
q_model.save('./saved')
# Load configure and weights by lpot.utils
saved_model = load("./saved", model)
eval_func(saved_model)
from lpot import Benchmark
evaluator = Benchmark('ptq_yaml.yaml')
# Load configure and weights by lpot.model
evaluator.model = common.Model(model)
evaluator.b_dataloader = common.DataLoader(dataset)
results = evaluator()
evaluator.model = common.Model(model)
fp32_results = evaluator()
self.assertTrue(
(fp32_results['accuracy'][0] - results['accuracy'][0]) < 0.01)
def test_tuning_ipex(self):
from lpot import Quantization
model = torchvision.models.resnet18()
model = MODELS['pytorch_ipex'](model)
quantizer = Quantization('ipex_yaml.yaml')
dataset = quantizer.dataset('dummy', (100, 3, 256, 256), label=True)
quantizer.model = common.Model(model)
quantizer.calib_dataloader = common.DataLoader(dataset)
quantizer.eval_dataloader = common.DataLoader(dataset)
lpot_model = quantizer()
lpot_model.save("./saved")
new_model = MODELS['pytorch_ipex'](model.model, {
"workspace_path": "./saved"
})
new_model.model.to(ipex.DEVICE)
try:
script_model = torch.jit.script(new_model.model)
except:
script_model = torch.jit.trace(
new_model.model,
torch.randn(10, 3, 224, 224).to(ipex.DEVICE))
from lpot import Benchmark
evaluator = Benchmark('ipex_yaml.yaml')
evaluator.model = common.Model(script_model)
evaluator.b_dataloader = common.DataLoader(dataset)
results = evaluator()
def test_register_metric_postprocess(self):
import PIL.Image
image = np.array(PIL.Image.open(self.image_path))
resize_image = np.resize(image, (224, 224, 3))
mean = [123.68, 116.78, 103.94]
resize_image = resize_image - mean
images = np.expand_dims(resize_image, axis=0)
labels = [768]
from lpot import Benchmark, Quantization, common
from lpot.data.transforms.imagenet_transform import LabelShift
from lpot.metric.metric import TensorflowTopK
evaluator = Benchmark('fake_yaml.yaml')
evaluator.postprocess = common.Postprocess(LabelShift,
'label_benchmark',
label_shift=1)
evaluator.metric = common.Metric(TensorflowTopK, 'topk_benchmark')
evaluator.b_dataloader = common.DataLoader(
dataset=list(zip(images, labels)))
evaluator.model = self.pb_path
result = evaluator()
acc, batch_size, result_list = result['accuracy']
self.assertEqual(acc, 0.0)
quantizer = Quantization('fake_yaml.yaml')
quantizer.postprocess = common.Postprocess(LabelShift,
'label_quantize',
label_shift=1)
quantizer.metric = common.Metric(TensorflowTopK, 'topk_quantize')
evaluator = Benchmark('fake_yaml.yaml')
evaluator.metric = common.Metric(TensorflowTopK, 'topk_second')
evaluator.b_dataloader = common.DataLoader(
dataset=list(zip(images, labels)))
evaluator.model = self.pb_path
result = evaluator()
acc, batch_size, result_list = result['accuracy']
self.assertEqual(acc, 0.0)
def main(_):
arg_parser = ArgumentParser(description='Parse args')
arg_parser.add_argument("--input-graph",
help='Specify the slim model',
dest='input_graph')
arg_parser.add_argument("--output-graph",
help='Specify tune result model save dir',
dest='output_graph')
arg_parser.add_argument("--config", default=None, help="tuning config")
arg_parser.add_argument('--benchmark', dest='benchmark', action='store_true', help='run benchmark')
arg_parser.add_argument('--tune', dest='tune', action='store_true', help='use lpot to tune.')
args = arg_parser.parse_args()
factory = TFSlimNetsFactory()
# user specific model can register to slim net factory
input_shape = [None, 299, 299, 3]
factory.register('inception_v4', inception_v4, input_shape, inception_v4_arg_scope)
if args.tune:
from lpot import Quantization
quantizer = Quantization(args.config)
quantizer.model = args.input_graph
q_model = quantizer()
q_model.save(args.output_graph)
if args.benchmark:
from lpot import Benchmark
evaluator = Benchmark(args.config)
evaluator.model = args.input_graph
results = evaluator()
for mode, result in results.items():
acc, batch_size, result_list = result
latency = np.array(result_list).mean() / batch_size
print('\n{} mode benchmark result:'.format(mode))
print('Accuracy is {:.3f}'.format(acc))
print('Batch size = {}'.format(batch_size))
print('Latency: {:.3f} ms'.format(latency * 1000))
print('Throughput: {:.3f} images/sec'.format(1./ latency))
def benchmark_model(
input_graph: str,
config: str,
benchmark_mode: str,
framework: str,
datatype: str = "",
) -> List[Dict[str, Any]]:
"""Execute benchmark."""
from lpot import Benchmark, common
benchmark_results = []
if framework == "onnxrt":
import onnx
input_graph = onnx.load(input_graph)
evaluator = Benchmark(config)
evaluator.model = common.Model(input_graph)
results = evaluator()
for mode, result in results.items():
if benchmark_mode == mode:
log.info(f"Mode: {mode}")
acc, batch_size, result_list = result
latency = (sum(result_list) / len(result_list)) / batch_size
log.info(f"Batch size: {batch_size}")
if mode == "accuracy":
log.info(f"Accuracy: {acc:.3f}")
elif mode == "performance":
log.info(f"Latency: {latency * 1000:.3f} ms")
log.info(f"Throughput: {1. / latency:.3f} images/sec")
benchmark_results.append(
{
"precision": datatype,
"mode": mode,
"batch_size": batch_size,
"accuracy": acc,
"latency": latency * 1000,
"throughput": 1.0 / latency,
},
)
return benchmark_results
def main():
import lpot
from lpot import common
quantizer = lpot.Quantization('./conf.yaml')
quantizer.model = common.Model("./mobilenet_v1_1.0_224_frozen.pb")
quantized_model = quantizer()
# Optional, run benchmark
from lpot import Benchmark
evaluator = Benchmark('./conf.yaml')
evaluator.model = common.Model(quantized_model)
results = evaluator()
batch_size = 1
for mode, result in results.items():
acc, batch_size, result_list = result
latency = np.array(result_list).mean() / batch_size
print('Accuracy is {:.3f}'.format(acc))
print('Latency: {:.3f} ms'.format(latency * 1000))
def main(_):
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO)
if FLAGS.mode == 'benchmark':
from lpot import Benchmark
evaluator = Benchmark(FLAGS.config)
evaluator.model = FLAGS.input_model
results = evaluator()
for mode, result in results.items():
acc, batch_size, result_list = result
latency = np.array(result_list).mean() / batch_size
print('\n{} mode benchmark result:'.format(mode))
print('Accuracy is {:.3f}'.format(acc))
print('Batch size = {}'.format(batch_size))
print('Latency: {:.3f} ms'.format(latency * 1000))
print('Throughput: {:.3f} images/sec'.format(1./ latency))
elif FLAGS.mode == 'tune':
from lpot.quantization import Quantization
quantizer = Quantization(FLAGS.config)
quantizer.model = FLAGS.input_model
q_model = quantizer()
q_model.save(FLAGS.output_model)
def run(self):
if self.args.tune:
from lpot import Quantization
quantizer = Quantization(self.args.config)
quantizer.model = self.args.input_graph
q_model = quantizer()
q_model.save(self.args.output_model)
if self.args.benchmark:
from lpot import Benchmark
evaluator = Benchmark(self.args.config)
evaluator.model = self.args.input_graph
results = evaluator()
for mode, result in results.items():
acc, batch_size, result_list = result
latency = np.array(result_list).mean() / batch_size
print('\n{} mode benchmark result:'.format(mode))
print('Accuracy is {:.3f}'.format(acc))
print('Batch size = {}'.format(batch_size))
print('Latency: {:.3f} ms'.format(latency * 1000))
print('Throughput: {:.3f} images/sec'.format(1. / latency))
parser.add_argument(
'--tune',
action='store_true', \
default=False,
help="whether quantize the model"
)
parser.add_argument('--config', type=str, help="config yaml path")
parser.add_argument('--output_model', type=str, help="output model path")
args = parser.parse_args()
model = onnx.load(args.model_path)
if args.benchmark:
from lpot import Benchmark, common
evaluator = Benchmark(args.config)
evaluator.model = common.Model(model)
results = evaluator()
for mode, result in results.items():
acc, batch_size, result_list = result
latency = np.array(result_list).mean() / batch_size
print('\n{} mode benchmark result:'.format(mode))
print('Accuracy is {:.3f}'.format(acc))
print('Batch size = {}'.format(batch_size))
print('Latency: {:.3f} ms'.format(latency * 1000))
print('Throughput: {:.3f} images/sec'.format(batch_size * 1. /
latency))
if args.tune:
from lpot import Quantization, common