def _test_output_int8(self, model, x, kind_in_graph=None, kind_not_in_graph=None, prec=None): modelName = model.__class__.__name__ core.enable_auto_dnnl() core.enable_jit_opt() model = model.to(ipex.DEVICE).eval() x = x.to(ipex.DEVICE) x2 = x.clone() x3 = x.clone() script_fused_model = torch.jit.script(copy.deepcopy(model)) trace_fused_model = torch.jit.trace(copy.deepcopy(model), x3) with torch.no_grad(): # int8, native path int8_calibration(model, [x], "configure.json") int8_conf = ipex.AmpConf(torch.int8, "configure.json") with ipex.AutoMixPrecision(int8_conf): result = model(x) # int8, jit script path script_graph = script_fused_model.graph_for(x2) int8_calibration(script_fused_model, [x2], "configure.json") int8_conf = ipex.AmpConf(torch.int8, "configure.json") with ipex.AutoMixPrecision(int8_conf): fused_sresult = script_fused_model(x2) # int8, jit trace path trace_graph = trace_fused_model.graph_for(x3) int8_calibration(trace_fused_model, [x3], "configure.json") int8_conf = ipex.AmpConf(torch.int8, "configure.json") with ipex.AutoMixPrecision(int8_conf): fused_tresult = trace_fused_model(x3) os.remove('configure.json') self.assertEqual(fused_sresult, result, prec) self.assertEqual(fused_tresult, result, prec) # check if the fused node exists in the graph if kind_in_graph is not None: self.assertTrue( any(n.kind() == kind_in_graph for n in script_graph.nodes())) self.assertTrue( any(n.kind() == kind_in_graph for n in trace_graph.nodes())) # check if certain node does not exist in the graph if kind_not_in_graph is not None: self.assertTrue( all(n.kind() != kind_not_in_graph for n in script_graph.nodes())) self.assertTrue( all(n.kind() != kind_not_in_graph for n in trace_graph.nodes()))
def compare_fp32_int8(self, model, x): conf = ipex.AmpConf(torch.int8) with ipex.AutoMixPrecision(conf, running_mode='calibration'): ref = model(x) conf.save('configure.json') conf = ipex.AmpConf(torch.int8, 'configure.json') with ipex.AutoMixPrecision(conf, running_mode='inference'): y = model(x) self.assertTrue(ipex.core.is_int8_dil_tensor(y)) self.assertEqual(ref, y, prec=0.1) os.remove('configure.json')
def int8_calibration(model, data, dir): conf = ipex.AmpConf(torch.int8) with torch.no_grad(): for x in data: with ipex.AutoMixPrecision(conf, running_mode="calibration"): model(x) conf.save(dir)
def run_linear(auto_mix_conf=None): for i in range(3): if auto_mix_conf != None: with ipex.AutoMixPrecision(auto_mix_conf): LL(get_input()) else: LL(get_input())
def validate(val_loader, model, criterion, args, ipex_config_path=None): batch_time = AverageMeter('Time', ':6.3f') losses = AverageMeter('Loss', ':.4e') top1 = AverageMeter('Acc@1', ':6.2f') top5 = AverageMeter('Acc@5', ':6.2f') progress = ProgressMeter(len(val_loader), batch_time, losses, top1, top5, prefix='Test: ') # switch to evaluate mode model.eval() if args.ipex: if ipex_config_path is not None: conf = ipex.AmpConf(torch.int8, configure_file=ipex_config_path) else: conf = ipex.AmpConf(None) with torch.no_grad(): for i, (input, target) in enumerate(val_loader): if i >= args.warmup_iter: start = time.time() if args.gpu is not None: input = input.cuda(args.gpu, non_blocking=True) target = target.cuda(args.gpu, non_blocking=True) # compute output if args.ipex: with ipex.AutoMixPrecision(conf, running_mode='inference'): output = model(input.to(ipex.DEVICE)) target = target.to(ipex.DEVICE) else: output = model(input) loss = criterion(output, target) # measure accuracy and record loss acc1, acc5 = accuracy(output, target, topk=(1, 5)) losses.update(loss.item(), input.size(0)) top1.update(acc1[0], input.size(0)) top5.update(acc5[0], input.size(0)) # measure elapsed time if i >= args.warmup_iter: batch_time.update(time.time() - start) if i % args.print_freq == 0: progress.print(i) if args.iter > 0 and i >= (args.warmup_iter + args.iter - 1): break print('Batch size = %d' % args.batch_size) if args.batch_size == 1: print('Latency: %.3f ms' % (batch_time.avg * 1000)) print('Throughput: %.3f images/sec' % (args.batch_size / batch_time.avg)) # TODO: this should also be done with the ProgressMeter print('Accuracy: {top1:.5f} Accuracy@5 {top5:.5f}' .format(top1=(top1.avg / 100), top5=(top5.avg / 100))) return top1.avg
def _lstm_compare_fp32_int8(self, model, *args): conf = ipex.AmpConf(torch.int8) with ipex.AutoMixPrecision(conf, running_mode='calibration'): with torch.no_grad(): ref, hy_ref = model(*args) conf.save('configure.json') conf = ipex.AmpConf(torch.int8, 'configure.json') with ipex.AutoMixPrecision(conf, running_mode='inference'): with torch.no_grad(): y, hy = model(*args) self.assertTrue(ipex.core.is_int8_dil_tensor(y)) self.assertEqual(ref, y, prec=0.1) self.assertEqual(hy_ref[0], hy[0], prec=0.01) self.assertEqual(hy_ref[1], hy[1], prec=0.01) os.remove('configure.json')
def test_quantization_status(self): x = torch.randn((4, 5), dtype=torch.float32).to(device) model = torch.nn.Linear(5, 10, bias=True).float().to(device) model1 = copy.deepcopy(model) x1 = x.clone() conf = ipex.AmpConf(torch.int8) with ipex.AutoMixPrecision(conf, running_mode='calibration'): ref = model1(x1) conf.save('configure.json') conf = ipex.AmpConf(torch.int8, 'configure.json') with ipex.AutoMixPrecision(conf, running_mode='inference'): y = model1(x1) self.assertTrue(ipex.core.is_int8_dil_tensor(y)) jsonFile = open('configure.json', 'r') data = json.load(jsonFile) jsonFile.close() self.assertTrue(data[0]['quantized']) # check configure's change can works for calibration step, # we need use origin model, because after running inference # step, the model has beem quantized, after change quantized # to False, the output should be fp32, i.e. not be quantized. data[0]['quantized'] = False jsonFile = open('configure.json', "w+") jsonFile.write(json.dumps(data)) jsonFile.close() # use user's changed configure. model2 = copy.deepcopy(model) x2 = x.clone() conf = ipex.AmpConf(torch.int8, 'configure.json') with ipex.AutoMixPrecision(conf, running_mode='calibration'): ref = model2(x2) conf.save('configure.json') conf = ipex.AmpConf(torch.int8, 'configure.json') jsonFile = open('configure.json', 'r') data = json.load(jsonFile) jsonFile.close() self.assertFalse(data[0]['quantized']) with ipex.AutoMixPrecision(conf, running_mode='inference'): y = model2(x2) self.assertTrue(ipex.core.is_fp32_dil_tensor(y)) os.remove('configure.json')
def test_quantization_status(self): x = torch.randn((4, 5), dtype=torch.float32).to(device) model = torch.nn.Linear(5, 10, bias=True).float().to(device) conf = ipex.AmpConf(torch.int8) with ipex.AutoMixPrecision(conf, running_mode='calibration'): ref = model(x) conf.save('configure.json') conf = ipex.AmpConf(torch.int8, 'configure.json') with ipex.AutoMixPrecision(conf, running_mode='inference'): y = model(x) self.assertTrue(ipex.core.is_int8_dil_tensor(y)) jsonFile = open('configure.json', 'r') data = json.load(jsonFile) jsonFile.close() self.assertTrue(data[0]['quantized']) # check configure's change can works for calibration step, # need get fp32 tensor for quantized=False. data[0]['quantized'] = False jsonFile = open('configure.json', "w+") jsonFile.write(json.dumps(data)) jsonFile.close() # use user's changed configure. conf = ipex.AmpConf(torch.int8, 'configure.json') with ipex.AutoMixPrecision(conf, running_mode='calibration'): ref = model(x) conf.save('configure.json') conf = ipex.AmpConf(torch.int8, 'configure.json') jsonFile = open('configure.json', 'r') data = json.load(jsonFile) jsonFile.close() self.assertFalse(data[0]['quantized']) with ipex.AutoMixPrecision(conf, running_mode='inference'): y = model(x) self.assertTrue(ipex.core.is_fp32_dil_tensor(y)) os.remove('configure.json')