def test_torch_latency( device, model, model_name, batch_sizes, sequence_lengths, global_lengths, test_times, num_threads, verbose, ): if num_threads > 0: torch.set_num_threads(num_threads) results = [] for batch_size in batch_sizes: for sequence_length in sequence_lengths: for global_length in global_lengths: print( f"batch_size={batch_size} sequence_length={sequence_length} global_length={global_length}..." ) inputs: LongformerInputs = LongformerHelper.get_dummy_inputs( batch_size, sequence_length, global_length, device) input_list = inputs.to_list() _ = model(*input_list) runtimes = timeit.repeat(lambda: model(*input_list), repeat=test_times, number=1) result = { "engine": "torch", # TODO: test torchscript "version": torch.__version__, "device": "cuda", "optimizer": "", "precision": "fp32", "io_binding": "", "model_name": model_name, "description": model_name + " [torch]", "inputs": 3, "threads": num_threads, "batch_size": batch_size, "sequence_length": sequence_length, "global_length": global_length, "datetime": str(datetime.now()), "memory": "NA", "diff_max": 0, "diff_90_percentile": 0, "diff_95_percentile": 0, "diff_99_percentile": 0, "use_compact_memory": "NA", } result.update( benchmark_helper.get_latency_result(runtimes, batch_size)) print(result) results.append(result) return results
def inference(): session = benchmark_helper.create_onnxruntime_session(onnx_model_path, use_gpu=True, enable_all_optimization=True, num_threads=num_threads) dummy_inputs: LongforerInputs = LongformerHelper.get_dummy_inputs(batch_size, sequence_length, global_length, device) ort_inputs = dummy_inputs.get_ort_inputs() for _ in range(test_times): ort_outputs = session.run(None, ort_inputs)
def test_parity(device, model, ort_session, batch_size, sequence_length, global_length, verbose=True): print( f"Comparing Torch and ORT outputs for batch_size={batch_size} sequence_length={sequence_length} global_length={global_length}..." ) dummy_inputs: LongforerInputs = LongformerHelper.get_dummy_inputs(batch_size, sequence_length, global_length, device) ort_inputs = dummy_inputs.get_ort_inputs() ort_outputs = ort_session.run(None, ort_inputs) input_list = dummy_inputs.to_list() torch_outputs = model(*input_list) max_diff = np.amax(torch_outputs[0].cpu().numpy() - ort_outputs[0]) print(f"last_state max diff = {max_diff}") if verbose and (math.isnan(max_diff) or max_diff > 0.001): print("torch last_state:", torch_outputs[0]) print("ort last_state:", ort_outputs[0]) return max_diff
def inference(): # Update Arena strategy so that we can measure the minimum memory required cuda_provider_options = {"arena_extend_strategy": "kSameAsRequested"} provider_options = {"CUDAExecutionProvider": cuda_provider_options} session = benchmark_helper.create_onnxruntime_session( onnx_model_path, use_gpu=True, enable_all_optimization=True, num_threads=num_threads, provider_options=provider_options, ) dummy_inputs: LongformerInputs = LongformerHelper.get_dummy_inputs( batch_size, sequence_length, global_length, device) ort_inputs = dummy_inputs.get_ort_inputs() for _ in range(test_times): _ = session.run(None, ort_inputs)
def test_ort_latency(device, model, model_name, description, ort_session, batch_sizes, sequence_lengths, global_lengths, test_times, num_threads, optimizer=False, precision='fp32', validate_onnx=True, disable_io_binding=False, verbose=True): results = [] for batch_size in batch_sizes: for sequence_length in sequence_lengths: for global_length in global_lengths: assert global_length <= model.config.attention_window[ 0], "Limitation of current implementation: number of global token <= attention_window" print( f"Testing batch_size={batch_size} sequence_length={sequence_length} global_length={global_length} optimizer={optimizer}, precision={precision} io_binding={not disable_io_binding}..." ) dummy_inputs: LongforerInputs = LongformerHelper.get_dummy_inputs(batch_size, sequence_length, global_length, device) # Run OnnxRuntime ort_inputs = dummy_inputs.get_ort_inputs() if verbose: print(ort_inputs) # run one query for warm up ort_outputs = ort_session.run(None, ort_inputs) result_template = { "model_name": model_name, "description": description, "inputs": 3, "engine": "OnnxRuntime", "version": onnxruntime.__version__, "device": "cuda", "precision": precision, "optimizer": optimizer, "threads": num_threads, "batch_size": batch_size, "sequence_length": sequence_length, "global_length": global_length, "test_times": test_times, "datetime": str(datetime.now()), "memory": "", } if not disable_io_binding: max_last_state_size = max(batch_sizes) * max(sequence_lengths) * model.config.hidden_size max_pooler_size = max(batch_sizes) * max(sequence_lengths) result = benchmark_helper.inference_ort_with_io_binding( ort_session, ort_inputs, result_template=result_template, repeat_times=test_times, ort_output_names=["last_state", "pooler"], ort_outputs=ort_outputs, output_buffers=[], output_buffer_max_sizes=[max_last_state_size, max_pooler_size], batch_size=batch_size, device=device, data_type=np.longlong, #input data type ) else: result = benchmark_helper.inference_ort(ort_session, ort_inputs, result_template=result_template, repeat_times=test_times, batch_size=batch_size) if validate_onnx: max_diff = test_parity(device, model, ort_session, batch_size, sequence_length, global_length, verbose) result["description"] += f"(max_diff={max_diff})" results.append(result) return results
def test_ort_latency( device, model, model_name, description, ort_session, batch_sizes, sequence_lengths, global_lengths, test_times, num_threads, optimizer=False, precision="fp32", disable_io_binding=False, verbose=True, use_compact_memory=False, use_half4=False, disable_parity=False, ) -> List[Dict[str, Any]]: results = [] for batch_size in batch_sizes: for sequence_length in sequence_lengths: for global_length in global_lengths: assert ( global_length <= model.config.attention_window[0] ), "Limitation of current implementation: number of global token <= attention_window" logger.info( f"Testing batch_size={batch_size} sequence_length={sequence_length} global_length={global_length} " f"optimizer={optimizer}, precision={precision} io_binding={not disable_io_binding}..." ) dummy_inputs: LongformerInputs = LongformerHelper.get_dummy_inputs( batch_size, sequence_length, global_length, device) # Run OnnxRuntime ort_inputs = dummy_inputs.get_ort_inputs() if verbose: print(ort_inputs) # run one query for warm up ort_outputs = ort_session.run(None, ort_inputs) result_template = { "model_name": model_name, "description": description, "inputs": 3, "engine": "OnnxRuntime", "version": str(onnxruntime.__version__), "device": "cuda", "precision": str(precision), "optimizer": int(optimizer), "threads": int(num_threads), "batch_size": int(batch_size), "sequence_length": int(sequence_length), "global_length": int(global_length), "test_times": int(test_times), "datetime": str(datetime.now()), "memory": "", "diff_max": None, "diff_90_percentile": None, "diff_95_percentile": None, "diff_99_percentile": None, "use_compact_memory": use_compact_memory, "use_half4": use_half4, } if not disable_io_binding: max_last_state_size = max(batch_sizes) * max( sequence_lengths) * model.config.hidden_size max_pooler_size = max(batch_sizes) * max(sequence_lengths) result = benchmark_helper.inference_ort_with_io_binding( ort_session, ort_inputs, result_template=result_template, repeat_times=test_times, ort_output_names=["last_state", "pooler"], ort_outputs=ort_outputs, output_buffers=[], output_buffer_max_sizes=[ max_last_state_size, max_pooler_size ], batch_size=batch_size, device=device, data_type=np.longlong, # input data type ) else: result = benchmark_helper.inference_ort( ort_session, ort_inputs, result_template=result_template, repeat_times=test_times, batch_size=batch_size, ) # measure result difference between PyTorch and OnnxRuntime if not disable_parity: diff_results = [ test_parity( device, model, ort_session, batch_size, sequence_length, global_length, verbose, ) for _ in range(test_times) ] result["diff_max"] = max(diff_results) result["diff_90_percentile"] = np.percentile( diff_results, 90) result["diff_95_percentile"] = np.percentile( diff_results, 95) result["diff_99_percentile"] = np.percentile( diff_results, 99) results.append(result) return results