def main(args): statistics = None try: if args.number_streams is None: logger.warn(" -nstreams default value is determined automatically for a device. " "Although the automatic selection usually provides a reasonable performance, " "but it still may be non-optimal for some cases, for more information look at README. ") if args.report_type: statistics = StatisticsReport(StatisticsReport.Config(args.report_type, args.report_folder)) statistics.add_parameters(StatisticsReport.Category.COMMAND_LINE_PARAMETERS, get_command_line_arguments(sys.argv)) # ------------------------------ 2. Loading Inference Engine --------------------------------------------------- next_step() device_name = args.target_device.upper() benchmark = Benchmark(args.target_device, args.number_infer_requests, args.number_iterations, args.time, args.api_type) benchmark.add_extension(args.path_to_extension, args.path_to_cldnn_config) version = benchmark.get_version_info() logger.info(version) # --------------------- 3. Read the Intermediate Representation of the network --------------------------------- next_step() start_time = datetime.now() ie_network = read_network(args.path_to_model) duration_ms = "{:.2f}".format((datetime.now() - start_time).total_seconds() * 1000) if statistics: logger.info("Read network took {} ms".format(duration_ms)) statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ('read network time (ms)', duration_ms) ]) # --------------------- 4. Resizing network to match image sizes and given batch ------------------------------- next_step() if args.batch_size and args.batch_size != ie_network.batch_size: benchmark.reshape(ie_network, args.batch_size) batch_size = ie_network.batch_size logger.info('Network batch size: {}, precision: {}'.format(ie_network.batch_size, ie_network.precision)) # --------------------- 5. Configuring input of the model ------------------------------------------------------ next_step() config_network_inputs(ie_network) # --------------------- 6. Setting device configuration -------------------------------------------------------- next_step() benchmark.set_config(args.number_streams, args.api_type, args.number_threads, args.infer_threads_pinning) # --------------------- 7. Loading the model to the device ----------------------------------------------------- next_step() start_time = datetime.now() perf_counts = True if args.perf_counts or \ args.report_type in [ averageCntReport, detailedCntReport ] or \ args.exec_graph_path else False exe_network = benchmark.load_network(ie_network, perf_counts, args.number_infer_requests) duration_ms = "{:.2f}".format((datetime.now() - start_time).total_seconds() * 1000) if statistics: logger.info("Load network took {} ms".format(duration_ms)) statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ('load network time (ms)', duration_ms) ]) # --------------------- 8. Setting optimal runtime parameters -------------------------------------------------- next_step() # Number of requests infer_requests = exe_network.requests benchmark.nireq = len(infer_requests) # Iteration limit benchmark.niter = get_number_iterations(benchmark.niter, len(exe_network.requests), args.api_type) # ------------------------------------ 9. Creating infer requests and filling input blobs ---------------------- next_step() request_queue = InferRequestsQueue(infer_requests) path_to_input = os.path.abspath(args.path_to_input) if args.path_to_input else None requests_input_data = get_inputs(path_to_input, batch_size, ie_network.inputs, infer_requests) if statistics: statistics.add_parameters(StatisticsReport.Category.RUNTIME_CONFIG, [ ('topology', ie_network.name), ('target device', device_name), ('API', args.api_type), ('precision', str(ie_network.precision)), ('batch size', str(ie_network.batch_size)), ('number of iterations', str(benchmark.niter) if benchmark.niter else "0"), ('number of parallel infer requests', str(benchmark.nireq)), ('duration (ms)', str(get_duration_in_milliseconds(benchmark.duration_seconds))), ]) for nstreams in benchmark.device_number_streams.items(): statistics.add_parameters(StatisticsReport.Category.RUNTIME_CONFIG, [ ("number of {} streams".format(nstreams[0]), str(nstreams[1])), ]) # ------------------------------------ 10. Measuring performance ----------------------------------------------- output_string = process_help_inference_string(benchmark) next_step(output_string) progress_bar_total_count = 10000 if benchmark.niter and not benchmark.duration_seconds: progress_bar_total_count = benchmark.niter progress_bar = ProgressBar(progress_bar_total_count, args.stream_output, args.progress) fps, latency_ms, total_duration_sec, iteration = benchmark.infer(request_queue, requests_input_data, batch_size, progress_bar) # ------------------------------------ 11. Dumping statistics report ------------------------------------------- next_step() if args.exec_graph_path: dump_exec_graph(exe_network, args.exec_graph_path) if perf_counts: perfs_count_list = [] for ni in range(int(benchmark.nireq)): perfs_count_list.append(exe_network.requests[ni].get_perf_counts()) if args.perf_counts: print_perf_counters(perfs_count_list) if statistics: statistics.dump_performance_counters(perfs_count_list) if statistics: statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ('total execution time (ms)', '{:.2f}'.format(get_duration_in_milliseconds(total_duration_sec))), ('total number of iterations', str(iteration)), ]) if MULTI_DEVICE_NAME not in device_name: statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ('latency (ms)', '{:.2f}'.format(latency_ms)), ]) statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ('throughput', '{:.2f}'.format(fps)), ]) if statistics: statistics.dump() print('Count: {} iterations'.format(iteration)) print('Duration: {:.2f} ms'.format(get_duration_in_milliseconds(total_duration_sec))) if MULTI_DEVICE_NAME not in device_name: print('Latency: {:.2f} ms'.format(latency_ms)) print('Throughput: {:.2f} FPS'.format(fps)) del exe_network next_step.step_id = 0 except Exception as e: logger.exception(e) if statistics: statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ('error', str(e)), ]) statistics.dump()
def benchmark_app(args): try: # ------------------------------ 2. Loading Inference Engine --------------------------------------------------- next_step() benchmark = Benchmark(args.target_device, args.number_infer_requests, args.number_iterations, args.time, args.api_type) benchmark.add_extension(args.path_to_extension, args.path_to_cldnn_config) version = benchmark.get_version_info() logger.info(version) # --------------------- 3. Read the Intermediate Representation of the network --------------------------------- next_step() ie_network = read_network(args.path_to_model) # --------------------- 4. Resizing network to match image sizes and given batch ------------------------------- next_step() if args.batch_size and args.batch_size != ie_network.batch_size: benchmark.reshape(ie_network, args.batch_size) batch_size = ie_network.batch_size logger.info('Network batch size: %s, precision: %s', ie_network.batch_size, ie_network.precision) # --------------------- 5. Configuring input of the model ------------------------------------------------------ next_step() config_network_inputs(ie_network) # --------------------- 6. Setting device configuration -------------------------------------------------------- next_step() benchmark.set_config(args.number_streams, args.api_type, args.number_threads, args.infer_threads_pinning) # --------------------- 7. Loading the model to the device ----------------------------------------------------- next_step() perf_counts = False if args.perf_counts or args.exec_graph_path: perf_counts = True exe_network = benchmark.load_network(ie_network, perf_counts, args.number_infer_requests) # --------------------- 8. Setting optimal runtime parameters -------------------------------------------------- next_step() # Number of requests infer_requests = exe_network.requests benchmark.nireq = len(infer_requests) # Iteration limit benchmark.niter = get_number_iterations(benchmark.niter, len(exe_network.requests), args.api_type) # ------------------------------------ 9. Creating infer requests and filling input blobs ---------------------- next_step() request_queue = InferRequestsQueue(infer_requests) path_to_input = os.path.abspath(args.path_to_input) if args.path_to_input else None requests_input_data = get_inputs(path_to_input, batch_size, ie_network.inputs, infer_requests) # ------------------------------------ 10. Measuring performance ----------------------------------------------- output_string = process_help_inference_string(benchmark) next_step(output_string) progress_bar_total_count = 10000 if benchmark.niter and not benchmark.duration_seconds: progress_bar_total_count = benchmark.niter progress_bar = ProgressBar(progress_bar_total_count, args.stream_output, args.progress) benchmark.infer(request_queue, requests_input_data, batch_size, progress_bar) # ------------------------------------ 11. Dumping statistics report ------------------------------------------- next_step() if args.exec_graph_path: dump_exec_graph(exe_network, args.exec_graph_path) del exe_network next_step.step_id = 0 except Exception as exc: logger.exception(exc)