def get_origin_tree_commit(distance: int, verbose: bool = False) -> str: """Returns the hash for the commit with the given distance from top of the tree for the origin base branch.""" base_branch = get_required_env_var("BUILDKITE_PULL_REQUEST_BASE_BRANCH") execute_cmd_and_get_output( ['git', 'fetch', '--prune', '--', 'origin', base_branch], cwd=THIS_DIRECTORY, verbose=verbose) return get_git_commit_hash(f'origin/{base_branch}~{distance}', verbose)
def set_cpu_frequency_scaling_governor(governor: str): git_root = execute_cmd_and_get_output( ["git", "rev-parse", "--show-toplevel"]) cpu_script = os.path.join(git_root, "build_tools", "benchmarks", "set_android_scaling_governor.sh") android_path = adb_push_to_tmp_dir(cpu_script) adb_execute_as_root([android_path, governor])
def __run_benchmark(self, android_case_dir: str, tool_name: str, driver: str, results_filename: str, taskset: str): host_tool_path = os.path.join(self.config.normal_benchmark_tool_dir, tool_name) android_tool = self.__check_and_push_file(host_tool_path, NORMAL_TOOL_REL_DIR) cmd = [ "taskset", taskset, android_tool, f"--flagfile={MODEL_FLAGFILE_NAME}" ] if tool_name == "iree-benchmark-module": cmd.extend([ "--benchmark_format=json", "--benchmark_out_format=json", f"--benchmark_out='{os.path.basename(results_filename)}'", ]) if self.config.benchmark_min_time: cmd.extend([ f"--benchmark_min_time={self.config.benchmark_min_time}", ]) else: repetitions = get_benchmark_repetition_count(driver) cmd.extend([ f"--benchmark_repetitions={repetitions}", ]) result_json = adb_execute_and_get_output(cmd, android_case_dir, verbose=self.verbose) # Pull the result file back onto the host and set the filename for later # return. pull_cmd = [ "adb", "pull", os.path.join(ANDROID_TMP_DIR, android_case_dir, os.path.basename(results_filename)), results_filename ] execute_cmd_and_get_output(pull_cmd, verbose=self.verbose) if self.verbose: print(result_json)
def set_gpu_frequency_scaling_policy(policy: str): git_root = execute_cmd_and_get_output(["git", "rev-parse", "--show-toplevel"]) device_model = get_android_device_model() gpu_name = get_android_gpu_name() if device_model == "Pixel-6" or device_model == "Pixel-6-Pro": gpu_script = os.path.join(git_root, "build_tools", "benchmarks", "set_pixel6_gpu_scaling_policy.sh") elif gpu_name.lower().startswith("adreno"): gpu_script = os.path.join(git_root, "build_tools", "benchmarks", "set_adreno_gpu_scaling_policy.sh") else: raise RuntimeError( f"Unsupported device '{device_model}' for setting GPU scaling policy") android_path = adb_push_to_tmp_dir(gpu_script) adb_execute_as_root([android_path, policy])
def get_git_commit_info(commit: str, verbose: bool = False) -> Dict[str, str]: """Gets commit information dictory for the given commit.""" cmd = [ 'git', 'show', '--format=%H:::%h:::%an:::%ae:::%s', '--no-patch', commit ] info = execute_cmd_and_get_output(cmd, cwd=THIS_DIRECTORY, verbose=verbose) segments = info.split(':::') return { 'hash': segments[0], 'abbrevHash': segments[1], 'authorName': segments[2], 'authorEmail': segments[3], 'subject': segments[4], }
def adb_execute_and_get_output(cmd_args: Sequence[str], relative_dir: str = "", verbose: bool = False) -> str: """Executes command with adb shell. Switches to `relative_dir` relative to the android tmp directory before executing. Waits for completion and returns the command stdout. Args: cmd_args: a list containing the command to execute and its parameters relative_dir: the directory to execute the command in; relative to ANDROID_TMP_DIR. Returns: A string for the command output. """ cmd = ["adb", "shell"] cmd.extend(["cd", os.path.join(ANDROID_TMP_DIR, relative_dir)]) cmd.append("&&") cmd.extend(cmd_args) return execute_cmd_and_get_output(cmd, verbose=verbose)
def get_git_commit_hash(commit: str) -> str: return execute_cmd_and_get_output(['git', 'rev-parse', commit], cwd=os.path.dirname( os.path.realpath(__file__)))
def main(args): device_info = get_android_device_info() if args.verbose: print(device_info) if not args.normal_benchmark_tool_dir and not args.traced_benchmark_tool_dir: raise ValueError( "At least one of --normal_benchmark_tool_dir or --traced_benchmark_tool_dir should be specified." ) do_capture = args.traced_benchmark_tool_dir is not None if ((args.traced_benchmark_tool_dir is not None) != do_capture) or ( (args.trace_capture_tool is not None) != do_capture) or ( (args.capture_tarball is not None) != do_capture): raise ValueError( "The following 3 flags should be simultaneously all specified or all unspecified: --traced_benchmark_tool_dir, --trace_capture_tool, --capture_tarball" ) if device_info.cpu_abi.lower() not in CPU_ABI_TO_TARGET_ARCH_MAP: raise ValueError(f"Unrecognized CPU ABI: '{device_info.cpu_abi}'; " "need to update the map") if device_info.gpu_name.lower() not in GPU_NAME_TO_TARGET_ARCH_MAP: raise ValueError(f"Unrecognized GPU name: '{device_info.gpu_name}'; " "need to update the map") if args.pin_cpu_freq: set_cpu_frequency_scaling_governor("performance") atexit.register(set_cpu_frequency_scaling_governor, "schedutil") if args.pin_gpu_freq: set_gpu_frequency_scaling_policy("performance") atexit.register(set_gpu_frequency_scaling_policy, "default") previous_benchmarks = None previous_captures = None # Collect names of previous benchmarks and captures that should be skipped and # merged into the results. if args.continue_from_directory is not None: previous_benchmarks_dir = os.path.join(args.continue_from_directory, "benchmark-results") if os.path.isdir(previous_benchmarks_dir): previous_benchmarks = set( os.path.splitext(os.path.basename(p))[0] for p in os.listdir(previous_benchmarks_dir)) if do_capture: previous_captures_dir = os.path.join(args.continue_from_directory, "captures") if os.path.isdir(previous_captures_dir): previous_captures = set( os.path.splitext(os.path.basename(p))[0] for p in os.listdir(previous_captures_dir)) # Clear the benchmark directory on the Android device first just in case # there are leftovers from manual or failed runs. execute_cmd_and_get_output(["adb", "shell", "rm", "-rf", ANDROID_TMP_DIR], verbose=args.verbose) if not args.no_clean: # Clear the benchmark directory on the Android device. atexit.register(execute_cmd_and_get_output, ["adb", "shell", "rm", "-rf", ANDROID_TMP_DIR], verbose=args.verbose) # Also clear temporary directory on the host device. atexit.register(shutil.rmtree, args.tmp_dir) # Tracy client and server communicate over port 8086 by default. If we want # to capture traces along the way, forward port via adb. if do_capture: execute_cmd_and_get_output(["adb", "forward", "tcp:8086", "tcp:8086"], verbose=args.verbose) atexit.register(execute_cmd_and_get_output, ["adb", "forward", "--remove", "tcp:8086"], verbose=args.verbose) results = BenchmarkResults() commit = get_git_commit_hash("HEAD") results.set_commit(commit) args.tmp_dir = os.path.join(args.tmp_dir, commit) os.makedirs(args.tmp_dir, exist_ok=True) benchmarks, captures, errors = filter_and_run_benchmarks( device_info=device_info, root_build_dir=args.build_dir, driver_filter=args.driver_filter_regex, model_name_filter=args.model_name_regex, mode_filter=args.mode_regex, tmp_dir=args.tmp_dir, normal_benchmark_tool_dir=real_path_or_none( args.normal_benchmark_tool_dir), traced_benchmark_tool_dir=real_path_or_none( args.traced_benchmark_tool_dir), trace_capture_tool=real_path_or_none(args.trace_capture_tool), skip_benchmarks=previous_benchmarks, skip_captures=previous_captures, do_capture=do_capture, keep_going=args.keep_going, benchmark_min_time=args.benchmark_min_time, verbose=args.verbose) # Merge in previous benchmarks and captures. if previous_benchmarks: benchmarks.extend(f"{os.path.join(previous_benchmarks_dir, b)}.json" for b in previous_benchmarks) if do_capture and previous_captures: captures.extend(f"{os.path.join(previous_captures_dir, c)}.tracy" for c in previous_captures) for b in benchmarks: with open(b) as f: result_json_object = json.loads(f.read()) benchmark_info = BenchmarkInfo.from_device_info_and_name( device_info, os.path.splitext(os.path.basename(b))[0]) benchmark_run = BenchmarkRun(benchmark_info, result_json_object["context"], result_json_object["benchmarks"]) results.benchmarks.append(benchmark_run) if args.output is not None: with open(args.output, "w") as f: f.write(results.to_json_str()) if args.verbose: print(results.commit) print(results.benchmarks) if captures: # Put all captures in a tarball and remove the origial files. with tarfile.open(args.capture_tarball, "w:gz") as tar: for capture_filename in captures: tar.add(capture_filename) if errors: print("Benchmarking completed with errors", file=sys.stderr) raise RuntimeError(errors)
def run_benchmarks_for_category( device_info: DeviceInfo, root_benchmark_dir: str, benchmark_category_dir: str, benchmark_case_dirs: Sequence[str], tmp_dir: str, normal_benchmark_tool_dir: Optional[str] = None, traced_benchmark_tool_dir: Optional[str] = None, trace_capture_tool: Optional[str] = None, skip_benchmarks: Optional[Set[str]] = None, skip_captures: Optional[Set[str]] = None, do_capture: bool = False, keep_going: bool = False, benchmark_min_time: float = 0, verbose: bool = False, ) -> Tuple[Sequence[Tuple[Optional[str], Optional[str]]], Sequence[Exception]]: """Runs all benchmarks on the Android device and reports results and captures. Args: device_info: an DeviceInfo object. root_benchmark_dir: path to the benchmark suite within the root build dir benchmark_category_dir: the directory to a specific benchmark category. benchmark_case_dirs: a list of benchmark case directories. tmp_dir: path to temporary directory to which intermediate outputs should be stored. Separate "benchmark-results" and "captures" subdirectories will be created as necessary. normal_benchmark_tool_dir: the path to the normal benchmark tool directory. traced_benchmark_tool_dir: the path to the tracing-enabled benchmark tool directory. trace_capture_tool: the path to the tool for collecting captured traces. skip_benchmarks: names of benchmarks that should be skipped. Note that captures will still be run for these benchmarks if do_capture is true and they are not also in skip_captures. skip_captures: names of benchmark captures that should be skipped. do_capture: whether captures should be collected. keep_going: whether to proceed if an individual run fails. Exceptions will logged and returned. benchmark_min_time: min number of seconds to run the benchmark for, if specified. Otherwise, the benchmark will be repeated a fixed number of times. verbose: whether to print additional debug information. Returns: A tuple with a list containing (benchmark-filename, capture-filename) tuples and a list containing raised exceptions (only if keep_going is true) """ push_vmfb_files( benchmark_case_dirs=benchmark_case_dirs, root_benchmark_dir=root_benchmark_dir, verbose=verbose, ) # Create directories on the host to store results from each benchmark run. benchmark_results_dir = os.path.join(tmp_dir, "benchmark-results") os.makedirs(benchmark_results_dir, exist_ok=True) # And the same for captures, if we are collecting them. captures_dir = os.path.join(tmp_dir, "captures") if do_capture: os.makedirs(captures_dir, exist_ok=True) results = [] errors = [] skip_benchmarks = skip_benchmarks if skip_benchmarks else set() skip_captures = skip_captures if skip_captures else set() # Push all model artifacts to the device and run them. root_benchmark_dir = os.path.dirname(benchmark_category_dir) for benchmark_case_dir in benchmark_case_dirs: # Read the file specifying which tool should be used for benchmarking with open(os.path.join(benchmark_case_dir, MODEL_TOOLFILE_NAME)) as f: tool = f.read().strip() if normal_benchmark_tool_dir: adb_push_to_tmp_dir(os.path.join(normal_benchmark_tool_dir, tool), relative_dir=NORMAL_TOOL_REL_DIR, verbose=verbose) if do_capture: adb_push_to_tmp_dir(os.path.join(traced_benchmark_tool_dir, tool), relative_dir=TRACED_TOOL_REL_DIR, verbose=verbose) benchmark_info = compose_info_object(device_info, benchmark_category_dir, benchmark_case_dir) benchmark_key = str(benchmark_info) # If we're not running the benchmark or the capture, just skip ahead. # No need to push files. if (benchmark_key in skip_benchmarks) and (not do_capture or benchmark_key in skip_captures): continue print(f"--> benchmark: {benchmark_info} <--") # Now try to actually run benchmarks and collect captures. If keep_going is # True then errors in the underlying commands will be logged and returned. try: android_relative_dir = os.path.relpath(benchmark_case_dir, root_benchmark_dir) adb_push_to_tmp_dir(os.path.join(benchmark_case_dir, MODEL_FLAGFILE_NAME), android_relative_dir, verbose=verbose) benchmark_result_filename = None if normal_benchmark_tool_dir and benchmark_key not in skip_benchmarks: benchmark_results_basename = f"{benchmark_key}.json" cmd = [ "taskset", benchmark_info.deduce_taskset(), os.path.join(ANDROID_TMP_DIR, NORMAL_TOOL_REL_DIR, tool), f"--flagfile={MODEL_FLAGFILE_NAME}" ] if tool == "iree-benchmark-module": cmd.extend([ "--benchmark_format=json", "--benchmark_out_format=json", f"--benchmark_out='{benchmark_results_basename}'", ]) if benchmark_min_time: cmd.extend([ f"--benchmark_min_time={benchmark_min_time}", ]) else: repetitions = get_benchmark_repetition_count( benchmark_info.runner) cmd.extend([ f"--benchmark_repetitions={repetitions}", ]) result_json = adb_execute_and_get_output(cmd, android_relative_dir, verbose=verbose) # Pull the result file back onto the host and set the filename for later # return. benchmark_result_filename = os.path.join( benchmark_results_dir, benchmark_results_basename) pull_cmd = [ "adb", "pull", os.path.join(ANDROID_TMP_DIR, android_relative_dir, benchmark_results_basename), benchmark_result_filename ] execute_cmd_and_get_output(pull_cmd, verbose=verbose) if verbose: print(result_json) capture_filename = None if do_capture and benchmark_key not in skip_captures: run_cmd = [ "TRACY_NO_EXIT=1", "taskset", benchmark_info.deduce_taskset(), os.path.join(ANDROID_TMP_DIR, TRACED_TOOL_REL_DIR, tool), f"--flagfile={MODEL_FLAGFILE_NAME}" ] # Just launch the traced benchmark tool with TRACY_NO_EXIT=1 without # waiting for the adb command to complete as that won't happen. process = adb_start_cmd(run_cmd, android_relative_dir, verbose=verbose) # But we do need to wait for its start; otherwise will see connection # failure when opening the catpure tool. Here we cannot just sleep a # certain amount of seconds---Pixel 4 seems to have an issue that will # make the trace collection step get stuck. Instead wait for the # benchmark result to be available. while True: line = process.stdout.readline() # pytype: disable=attribute-error if line == "" and process.poll( ) is not None: # Process completed raise ValueError( "Cannot find benchmark result line in the log!") if verbose: print(line.strip()) # Result available if re.match(r"^BM_.+/real_time", line) is not None: break # Now it's okay to collect the trace via the capture tool. This will # send the signal to let the previously waiting benchmark tool to # complete. capture_filename = os.path.join(captures_dir, f"{benchmark_key}.tracy") capture_cmd = [ trace_capture_tool, "-f", "-o", capture_filename ] capture_log = execute_cmd_and_get_output(capture_cmd, verbose=verbose) if verbose: print(capture_log) print("...benchmark completed") results.append((benchmark_result_filename, capture_filename)) time.sleep(1) # Some grace time. except subprocess.CalledProcessError as e: if keep_going: print(f"Processing of benchmark failed with: {e}") errors.append(e) continue raise e return (results, errors)
def get_git_total_commit_count(commit: str, verbose: bool = False) -> int: """Gets the total commit count in history ending with the given commit.""" count = execute_cmd_and_get_output(['git', 'rev-list', '--count', commit], cwd=THIS_DIRECTORY, verbose=verbose) return int(count)
def get_git_commit_hash(commit: str, verbose: bool = False) -> str: """Gets the commit hash for the given commit.""" return execute_cmd_and_get_output(['git', 'rev-parse', commit], cwd=THIS_DIRECTORY, verbose=verbose)
def main(args): device_info = get_android_device_info(args.verbose) if args.verbose: print(device_info) commit = get_git_commit_hash("HEAD") benchmark_config = BenchmarkConfig.build_from_args(args, commit) benchmark_suite = BenchmarkSuite.load_from_benchmark_suite_dir( benchmark_config.root_benchmark_dir) benchmark_driver = AndroidBenchmarkDriver(device_info=device_info, benchmark_config=benchmark_config, benchmark_suite=benchmark_suite, benchmark_grace_time=1.0, verbose=args.verbose) if args.continue_from_directory: # Merge in previous benchmarks and captures. benchmark_driver.add_previous_benchmarks_and_captures( args.continue_from_directory) if args.pin_cpu_freq: set_cpu_frequency_scaling_governor("performance") atexit.register(set_cpu_frequency_scaling_governor, "schedutil") if args.pin_gpu_freq: set_gpu_frequency_scaling_policy("performance") atexit.register(set_gpu_frequency_scaling_policy, "default") # Clear the benchmark directory on the Android device first just in case # there are leftovers from manual or failed runs. execute_cmd_and_get_output(["adb", "shell", "rm", "-rf", ANDROID_TMP_DIR], verbose=args.verbose) if not args.no_clean: # Clear the benchmark directory on the Android device. atexit.register(execute_cmd_and_get_output, ["adb", "shell", "rm", "-rf", ANDROID_TMP_DIR], verbose=args.verbose) # Also clear temporary directory on the host device. atexit.register(shutil.rmtree, args.tmp_dir) # Tracy client and server communicate over port 8086 by default. If we want # to capture traces along the way, forward port via adb. trace_capture_config = benchmark_config.trace_capture_config if trace_capture_config: execute_cmd_and_get_output(["adb", "forward", "tcp:8086", "tcp:8086"], verbose=args.verbose) atexit.register(execute_cmd_and_get_output, ["adb", "forward", "--remove", "tcp:8086"], verbose=args.verbose) benchmark_driver.run() benchmark_results = benchmark_driver.get_benchmark_results() if args.output is not None: with open(args.output, "w") as f: f.write(benchmark_results.to_json_str()) if args.verbose: print(benchmark_results.commit) print(benchmark_results.benchmarks) if trace_capture_config: # Put all captures in a tarball and remove the origial files. with tarfile.open(trace_capture_config.capture_tarball, "w:gz") as tar: for capture_filename in benchmark_driver.get_capture_filenames(): tar.add(capture_filename) benchmark_errors = benchmark_driver.get_benchmark_errors() if benchmark_errors: print("Benchmarking completed with errors", file=sys.stderr) raise RuntimeError(benchmark_errors)