def _test_output_int8(self,
model,
x,
kind_in_graph=None,
kind_not_in_graph=None,
prec=None):
modelName = model.__class__.__name__
core.enable_auto_dnnl()
core.enable_jit_opt()
model = model.to(ipex.DEVICE).eval()
x = x.to(ipex.DEVICE)
x2 = x.clone()
x3 = x.clone()
script_fused_model = torch.jit.script(copy.deepcopy(model))
trace_fused_model = torch.jit.trace(copy.deepcopy(model), x3)
with torch.no_grad():
# int8, native path
int8_calibration(model, [x], "configure.json")
int8_conf = ipex.AmpConf(torch.int8, "configure.json")
with ipex.AutoMixPrecision(int8_conf):
result = model(x)
# int8, jit script path
script_graph = script_fused_model.graph_for(x2)
int8_calibration(script_fused_model, [x2], "configure.json")
int8_conf = ipex.AmpConf(torch.int8, "configure.json")
with ipex.AutoMixPrecision(int8_conf):
fused_sresult = script_fused_model(x2)
# int8, jit trace path
trace_graph = trace_fused_model.graph_for(x3)
int8_calibration(trace_fused_model, [x3], "configure.json")
int8_conf = ipex.AmpConf(torch.int8, "configure.json")
with ipex.AutoMixPrecision(int8_conf):
fused_tresult = trace_fused_model(x3)
os.remove('configure.json')
self.assertEqual(fused_sresult, result, prec)
self.assertEqual(fused_tresult, result, prec)
# check if the fused node exists in the graph
if kind_in_graph is not None:
self.assertTrue(
any(n.kind() == kind_in_graph for n in script_graph.nodes()))
self.assertTrue(
any(n.kind() == kind_in_graph for n in trace_graph.nodes()))
# check if certain node does not exist in the graph
if kind_not_in_graph is not None:
self.assertTrue(
all(n.kind() != kind_not_in_graph
for n in script_graph.nodes()))
self.assertTrue(
all(n.kind() != kind_not_in_graph
for n in trace_graph.nodes()))
def compare_fp32_int8(self, model, x):
conf = ipex.AmpConf(torch.int8)
with ipex.AutoMixPrecision(conf, running_mode='calibration'):
ref = model(x)
conf.save('configure.json')
conf = ipex.AmpConf(torch.int8, 'configure.json')
with ipex.AutoMixPrecision(conf, running_mode='inference'):
y = model(x)
self.assertTrue(ipex.core.is_int8_dil_tensor(y))
self.assertEqual(ref, y, prec=0.1)
os.remove('configure.json')
def int8_calibration(model, data, dir):
conf = ipex.AmpConf(torch.int8)
with torch.no_grad():
for x in data:
with ipex.AutoMixPrecision(conf, running_mode="calibration"):
model(x)
conf.save(dir)
def run_linear(auto_mix_conf=None):
for i in range(3):
if auto_mix_conf != None:
with ipex.AutoMixPrecision(auto_mix_conf):
LL(get_input())
else:
LL(get_input())
def validate(val_loader, model, criterion, args, ipex_config_path=None):
batch_time = AverageMeter('Time', ':6.3f')
losses = AverageMeter('Loss', ':.4e')
top1 = AverageMeter('Acc@1', ':6.2f')
top5 = AverageMeter('Acc@5', ':6.2f')
progress = ProgressMeter(len(val_loader), batch_time, losses, top1, top5,
prefix='Test: ')
# switch to evaluate mode
model.eval()
if args.ipex:
if ipex_config_path is not None:
conf = ipex.AmpConf(torch.int8, configure_file=ipex_config_path)
else:
conf = ipex.AmpConf(None)
with torch.no_grad():
for i, (input, target) in enumerate(val_loader):
if i >= args.warmup_iter:
start = time.time()
if args.gpu is not None:
input = input.cuda(args.gpu, non_blocking=True)
target = target.cuda(args.gpu, non_blocking=True)
# compute output
if args.ipex:
with ipex.AutoMixPrecision(conf, running_mode='inference'):
output = model(input.to(ipex.DEVICE))
target = target.to(ipex.DEVICE)
else:
output = model(input)
loss = criterion(output, target)
# measure accuracy and record loss
acc1, acc5 = accuracy(output, target, topk=(1, 5))
losses.update(loss.item(), input.size(0))
top1.update(acc1[0], input.size(0))
top5.update(acc5[0], input.size(0))
# measure elapsed time
if i >= args.warmup_iter:
batch_time.update(time.time() - start)
if i % args.print_freq == 0:
progress.print(i)
if args.iter > 0 and i >= (args.warmup_iter + args.iter - 1):
break
print('Batch size = %d' % args.batch_size)
if args.batch_size == 1:
print('Latency: %.3f ms' % (batch_time.avg * 1000))
print('Throughput: %.3f images/sec' % (args.batch_size / batch_time.avg))
# TODO: this should also be done with the ProgressMeter
print('Accuracy: {top1:.5f} Accuracy@5 {top5:.5f}'
.format(top1=(top1.avg / 100), top5=(top5.avg / 100)))
return top1.avg
def _lstm_compare_fp32_int8(self, model, *args):
conf = ipex.AmpConf(torch.int8)
with ipex.AutoMixPrecision(conf, running_mode='calibration'):
with torch.no_grad():
ref, hy_ref = model(*args)
conf.save('configure.json')
conf = ipex.AmpConf(torch.int8, 'configure.json')
with ipex.AutoMixPrecision(conf, running_mode='inference'):
with torch.no_grad():
y, hy = model(*args)
self.assertTrue(ipex.core.is_int8_dil_tensor(y))
self.assertEqual(ref, y, prec=0.1)
self.assertEqual(hy_ref[0], hy[0], prec=0.01)
self.assertEqual(hy_ref[1], hy[1], prec=0.01)
os.remove('configure.json')
def test_quantization_status(self):
x = torch.randn((4, 5), dtype=torch.float32).to(device)
model = torch.nn.Linear(5, 10, bias=True).float().to(device)
model1 = copy.deepcopy(model)
x1 = x.clone()
conf = ipex.AmpConf(torch.int8)
with ipex.AutoMixPrecision(conf, running_mode='calibration'):
ref = model1(x1)
conf.save('configure.json')
conf = ipex.AmpConf(torch.int8, 'configure.json')
with ipex.AutoMixPrecision(conf, running_mode='inference'):
y = model1(x1)
self.assertTrue(ipex.core.is_int8_dil_tensor(y))
jsonFile = open('configure.json', 'r')
data = json.load(jsonFile)
jsonFile.close()
self.assertTrue(data[0]['quantized'])
# check configure's change can works for calibration step,
# we need use origin model, because after running inference
# step, the model has beem quantized, after change quantized
# to False, the output should be fp32, i.e. not be quantized.
data[0]['quantized'] = False
jsonFile = open('configure.json', "w+")
jsonFile.write(json.dumps(data))
jsonFile.close()
# use user's changed configure.
model2 = copy.deepcopy(model)
x2 = x.clone()
conf = ipex.AmpConf(torch.int8, 'configure.json')
with ipex.AutoMixPrecision(conf, running_mode='calibration'):
ref = model2(x2)
conf.save('configure.json')
conf = ipex.AmpConf(torch.int8, 'configure.json')
jsonFile = open('configure.json', 'r')
data = json.load(jsonFile)
jsonFile.close()
self.assertFalse(data[0]['quantized'])
with ipex.AutoMixPrecision(conf, running_mode='inference'):
y = model2(x2)
self.assertTrue(ipex.core.is_fp32_dil_tensor(y))
os.remove('configure.json')
def test_quantization_status(self):
x = torch.randn((4, 5), dtype=torch.float32).to(device)
model = torch.nn.Linear(5, 10, bias=True).float().to(device)
conf = ipex.AmpConf(torch.int8)
with ipex.AutoMixPrecision(conf, running_mode='calibration'):
ref = model(x)
conf.save('configure.json')
conf = ipex.AmpConf(torch.int8, 'configure.json')
with ipex.AutoMixPrecision(conf, running_mode='inference'):
y = model(x)
self.assertTrue(ipex.core.is_int8_dil_tensor(y))
jsonFile = open('configure.json', 'r')
data = json.load(jsonFile)
jsonFile.close()
self.assertTrue(data[0]['quantized'])
# check configure's change can works for calibration step,
# need get fp32 tensor for quantized=False.
data[0]['quantized'] = False
jsonFile = open('configure.json', "w+")
jsonFile.write(json.dumps(data))
jsonFile.close()
# use user's changed configure.
conf = ipex.AmpConf(torch.int8, 'configure.json')
with ipex.AutoMixPrecision(conf, running_mode='calibration'):
ref = model(x)
conf.save('configure.json')
conf = ipex.AmpConf(torch.int8, 'configure.json')
jsonFile = open('configure.json', 'r')
data = json.load(jsonFile)
jsonFile.close()
self.assertFalse(data[0]['quantized'])
with ipex.AutoMixPrecision(conf, running_mode='inference'):
y = model(x)
self.assertTrue(ipex.core.is_fp32_dil_tensor(y))
os.remove('configure.json')