def main(argv): """Main entry point.""" argv = FLAGS(argv) if len(argv) != 1: raise app.UsageError(f"Unknown command line arguments: {argv[1:]}") # Use default logdir of <base>/brute_force/<benchmark> unless told # otherwise. benchmark = benchmark_from_flags() if not benchmark: raise app.UsageError("No benchmark specified.") env = env_from_flags(benchmark) env.reset() benchmark = env.benchmark sanitized_benchmark_name = "/".join(benchmark.split("/")[-2:]) env.close() logs_dir = Path( FLAGS.output_dir or create_logging_dir(f"brute_force/{sanitized_benchmark_name}")) run_brute_force( make_env=lambda: env_from_flags(benchmark_from_flags()), action_names=FLAGS.actions, episode_length=FLAGS.episode_length, outdir=logs_dir, nproc=FLAGS.nproc, )
def main(argv): """Main entry point.""" argv = FLAGS(argv) if len(argv) != 1: raise app.UsageError(f"Unknown command line arguments: {argv[1:]}") # Use default logdir of <base>/brute_force/<benchmark> unless told # otherwise. benchmark = benchmark_from_flags() if not benchmark: raise app.UsageError("No benchmark specified.") with env_from_flags(benchmark) as env: env.reset() logs_dir = Path(FLAGS.output_dir or create_logging_dir( f'brute_force/{os.path.normpath(f"random/{env.benchmark.uri.scheme}/{env.benchmark.uri.path}")}' )) run_brute_force( make_env=lambda: env_from_flags(benchmark_from_flags()), action_names=FLAGS.brute_force_action_list, episode_length=FLAGS.episode_length, outdir=logs_dir, nproc=FLAGS.nproc, )
def main(argv): """Main entry point.""" argv = FLAGS(argv) if len(argv) != 1: raise app.UsageError(f"Unknown command line arguments: {argv[1:]}") # Determine the benchmark that is being analyzed, or use all of them. benchmark = benchmark_from_flags() if benchmark: benchmarks = [benchmark] else: with env_from_flags() as env: benchmarks = islice(env.benchmarks, 100) logs_dir = Path(FLAGS.output_dir or create_logging_dir("benchmark_sensitivity_analysis")) rewards_path = logs_dir / f"benchmarks_{FLAGS.reward}.csv" runtimes_path = logs_dir / f"benchmarks_{FLAGS.reward}_runtimes.csv" run_benchmark_sensitivity_analysis( rewards_path=rewards_path, runtimes_path=runtimes_path, benchmarks=benchmarks, reward=FLAGS.reward, num_trials=FLAGS.num_benchmark_sensitivity_trials, min_steps=FLAGS.min_steps, max_steps=FLAGS.max_steps, nproc=FLAGS.nproc, max_attempts_multiplier=FLAGS.max_benchmark_attempts_multiplier, )
def get_rewards( benchmark: Union[Benchmark, str], reward_space: str, num_trials: int, min_steps: int, max_steps: int, max_attempts_multiplier: int = 5, ) -> SensitivityAnalysisResult: """Run random trials to get a list of num_trials episode rewards.""" rewards, runtimes = [], [] num_attempts = 0 while (num_attempts < max_attempts_multiplier * num_trials and len(rewards) < num_trials): num_attempts += 1 with env_from_flags(benchmark=benchmark) as env: env.observation_space = None env.reward_space = None env.reset(benchmark=benchmark) benchmark = env.benchmark with Timer() as t: reward = run_one_trial(env, reward_space, min_steps, max_steps) if reward is not None: rewards.append(reward) runtimes.append(t.time) return SensitivityAnalysisResult(name=env.benchmark, runtimes=np.array(runtimes), rewards=np.array(rewards))
def run_action_sensitivity_analysis( actions: List[int], rewards_path: Path, runtimes_path: Path, reward_space: str, num_trials: int, max_warmup_steps: int, nproc: int = cpu_count(), max_attempts_multiplier: int = 5, ): """Estimate the immediate reward of a given list of actions.""" with env_from_flags() as env: action_names = env.action_space.names with ThreadPoolExecutor(max_workers=nproc) as executor: analysis_futures = { executor.submit( get_rewards, action, action_names[action], reward_space, num_trials, max_warmup_steps, max_attempts_multiplier, ) for action in actions } return run_sensitivity_analysis( analysis_futures=analysis_futures, runtimes_path=runtimes_path, rewards_path=rewards_path, )
def main(argv): """Main entry point.""" argv = FLAGS(argv) if len(argv) != 1: raise app.UsageError(f"Unknown command line arguments: {argv[1:]}") with env_from_flags() as env: action_names = env.action_space.names if FLAGS.action: actions = [action_names.index(a) for a in FLAGS.action] else: actions = list(range(len(action_names))) logs_dir = Path( FLAGS.output_dir or create_logging_dir("benchmark_sensitivity_analysis") ) rewards_path = logs_dir / f"actions_{FLAGS.reward}.rewards.csv" runtimes_path = logs_dir / f"actions_{FLAGS.reward}.runtimes.csv" run_action_sensitivity_analysis( rewards_path=rewards_path, runtimes_path=runtimes_path, actions=actions, reward=FLAGS.reward, num_trials=FLAGS.num_action_sensitivity_trials, max_warmup_steps=FLAGS.max_warmup_steps, nproc=FLAGS.nproc, max_attempts_multiplier=FLAGS.max_action_attempts_multiplier, )
def get_rewards( action: int, action_name: str, reward_space: str, num_trials: int, max_warmup_steps: int, max_attempts_multiplier: int = 5, ) -> SensitivityAnalysisResult: """Run random trials to get a list of num_trials immediate rewards.""" rewards, runtimes = [], [] benchmark = benchmark_from_flags() num_attempts = 0 while ( num_attempts < max_attempts_multiplier * num_trials and len(rewards) < num_trials ): num_attempts += 1 with env_from_flags(benchmark=benchmark) as env: env.observation_space = None env.reward_space = None env.reset(benchmark=benchmark) with Timer() as t: reward = run_one_trial(env, reward_space, action, max_warmup_steps) if reward is not None: rewards.append(reward) runtimes.append(t.time) return SensitivityAnalysisResult( name=action_name, runtimes=np.array(runtimes), rewards=np.array(rewards) )
def main(argv): """Main entry point.""" argv = FLAGS(argv) if len(argv) != 1: raise app.UsageError(f"Unknown command line arguments: {argv[1:]}") if FLAGS.ls_benchmark: env = env_from_flags() print("\n".join(sorted(env.benchmarks))) env.close() return if FLAGS.ls_reward: env = env_from_flags() print("\n".join(sorted(env.reward.indices.keys()))) env.close() return assert FLAGS.patience >= 0, "--patience must be >= 0" def make_env(): return env_from_flags(benchmark=benchmark_from_flags()) env = make_env() try: env.reset() if not env.benchmark: raise app.UsageError("No benchmark specified.") finally: env.close() best_reward, _ = random_search( make_env=make_env, outdir=Path(FLAGS.output_dir) if FLAGS.output_dir else None, patience=FLAGS.patience, total_runtime=FLAGS.runtime, nproc=FLAGS.nproc, skip_done=FLAGS.skip_done, ) # Exit with error if --fail_threshold was set and the best reward does not # meet this value. if FLAGS.fail_threshold is not None and best_reward < FLAGS.fail_threshold: print( f"Best reward {best_reward:.3f} below threshold of {FLAGS.fail_threshold}", file=sys.stderr, ) sys.exit(1)
def main(argv): """Main entry point.""" assert len(argv) == 1, f"Unrecognized flags: {argv[1:]}" assert 0 < FLAGS.heading_level <= 4, f"--heading_level must be in range [1,4]" env = env_from_flags() print_service_capabilities(env, base_heading_level=FLAGS.heading_level) env.close()
def main(argv): """Main entry point.""" assert len(argv) == 1, f"Unrecognized flags: {argv[1:]}" with env_from_flags(benchmark=benchmark_from_flags()) as env: step_min = min(FLAGS.step_min, FLAGS.step_max) step_max = max(FLAGS.step_min, FLAGS.step_max) run_random_walk(env=env, step_count=random.randint(step_min, step_max))
def run_random_search(num_episodes, num_steps) -> None: """The inner loop of a load test benchmark.""" with env_from_flags(benchmark=benchmark_from_flags()) as env: for _ in range(num_episodes): env.reset() for _ in range(num_steps): _, _, done, _ = env.step(env.action_space.sample()) if done: break
def make_env(): FLAGS.env = "llvm-v0" if not FLAGS.reward: FLAGS.reward = "IrInstructionCountOz" env = env_from_flags(benchmark=benchmark_from_flags()) env = ConstrainedCommandline(env, flags=FLAGS.flags) env = TimeLimit(env, max_episode_steps=FLAGS.episode_len) env = HistoryObservation(env) return env
def main(argv): """Main entry point.""" argv = FLAGS(argv) if len(argv) != 1: raise app.UsageError(f"Unknown command line arguments: {argv[1:]}") if FLAGS.ls_benchmark: benchmark = benchmark_from_flags() env = env_from_flags(benchmark) print("\n".join(sorted(env.benchmarks))) env.close() return with Timer("Initialized environment"): benchmark = benchmark_from_flags() env = env_from_flags(benchmark) run_manual_env(env)
def main(argv): """Main entry point.""" argv = FLAGS(argv) if len(argv) != 1: raise app.UsageError(f"Unknown command line arguments: {argv[1:]}") if FLAGS.ls_benchmark: benchmark = benchmark_from_flags() env = env_from_flags(benchmark) print("\n".join(sorted(env.benchmarks))) env.close() return with Timer("Initialized environment"): # FIXME Chris, I don't seem to actually get a benchmark benchmark = benchmark_from_flags() env = env_from_flags(benchmark) shell = CompilerGymShell(env) shell.cmdloop()
def main(argv): """Main entry point.""" argv = FLAGS(argv) if len(argv) != 1: raise app.UsageError(f"Unknown command line arguments: {argv[1:]}") with Timer("Initialized environment"): benchmark = benchmark_from_flags() env = env_from_flags(benchmark) shell = CompilerGymShell(env) shell.cmdloop()
def main(argv): """Main entry point.""" argv = FLAGS(argv) if len(argv) != 1: raise app.UsageError(f"Unknown command line arguments: {argv[1:]}") if FLAGS.ls_reward: with env_from_flags() as env: print("\n".join(sorted(env.reward.indices.keys()))) return assert FLAGS.patience >= 0, "--patience must be >= 0" # Create an environment now to catch a startup time error before we launch # a bunch of workers. with env_from_flags() as env: env.reset(benchmark=benchmark_from_flags()) env = random_search( make_env=lambda: env_from_flags(benchmark=benchmark_from_flags()), outdir=Path(FLAGS.output_dir) if FLAGS.output_dir else None, patience=FLAGS.patience, total_runtime=FLAGS.runtime, nproc=FLAGS.nproc, skip_done=FLAGS.skip_done, ) try: # Exit with error if --fail_threshold was set and the best reward does not # meet this value. if ( FLAGS.fail_threshold is not None and env.episode_reward < FLAGS.fail_threshold ): print( f"Best reward {env.episode_reward:.3f} below threshold of {FLAGS.fail_threshold}", file=sys.stderr, ) sys.exit(1) finally: env.close()
def main(argv): """Main entry point.""" argv = FLAGS(argv) if len(argv) != 1: raise app.UsageError(f"Unknown command line arguments: {argv[1:]}") output_dir = Path(FLAGS.output_dir).expanduser().resolve().absolute() assert ( output_dir / logs.METADATA_NAME ).is_file(), f"Invalid --output_dir: {output_dir}" env = env_from_flags() benchmark = benchmark_from_flags() replay_actions_from_logs(env, output_dir, benchmark=benchmark)
def main(argv): """Main entry point.""" assert len(argv) == 1, f"Unrecognized flags: {argv[1:]}" if FLAGS.run_on_port: assert FLAGS.env, "Must specify an --env to run" settings = ConnectionOpts( script_args=["--port", str(FLAGS.run_on_port)]) with gym.make(FLAGS.env, connection_settings=settings) as env: print( f"=== Started a service on port {FLAGS.run_on_port}. Use C-c to terminate. ===" ) signal.pause() with env_from_flags() as env: print_service_capabilities(env)
def __init__(self): self._env = env_from_flags(benchmark_from_flags()) try: # Project onto the subset of transformations that have # been specified to be used. if not FLAGS.actions: self._action_indices = list( range(len(self._env.action_space.names))) else: self._action_indices = [ self._env.action_space.flags.index(a) for a in FLAGS.actions ] self._action_names = [ self._env.action_space.names[a] for a in self._action_indices ] finally: # The program will not terminate until the environment is # closed, not even if there is an exception. self._env.close()
def main(argv): """Main entry point.""" if len(argv) != 1: raise app.UsageError(f"Unknown command line arguments: {argv[1:]}") with env_from_flags() as env: invalidated_manifest = False for name_or_url in FLAGS.download: env.datasets.install(name_or_url) if FLAGS.download_all: for dataset in env.datasets: dataset.install() for name in FLAGS.activate: activate(env, name) invalidated_manifest = True if FLAGS.activate_all: invalidated_manifest = True for name in FLAGS.deactivate: deactivate(env, name) invalidated_manifest = True if FLAGS.deactivate_all: invalidated_manifest = True for name in FLAGS.delete: delete(env, name) if invalidated_manifest: env.make_manifest_file() print( summarize_datasets(env.datasets), )
def make_env(): env = env_from_flags(benchmark=benchmark_from_flags()) if FLAGS.explore_actions: env = ConstrainedCommandline(env, flags=FLAGS.explore_actions) return env
def main(argv): """Main entry point.""" try: states = list(CompilerEnvStateReader.read_paths(argv[1:])) except ValueError as e: print(e, file=sys.stderr) sys.exit(1) if not states: print( "No inputs to validate. Pass a CSV file path as an argument, or " "use - to read from stdin.", file=sys.stderr, ) sys.exit(1) # Send the states off for validation if FLAGS.debug_force_valid: validation_results = ( ValidationResult( state=state, reward_validated=True, actions_replay_failed=False, reward_validation_failed=False, benchmark_semantics_validated=False, benchmark_semantics_validation_failed=False, walltime=0, ) for state in states ) else: validation_results = validate_states( env_from_flags, states, nproc=FLAGS.nproc, inorder=FLAGS.inorder, ) # Determine the name of the reward space. with env_from_flags() as env: if FLAGS.reward_aggregation == "geomean": def reward_aggregation(a): return geometric_mean(np.clip(a, 0, None)) reward_aggregation_name = "Geometric mean" elif FLAGS.reward_aggregation == "mean": reward_aggregation = arithmetic_mean reward_aggregation_name = "Mean" else: raise app.UsageError( f"Unknown aggregation type: '{FLAGS.reward_aggregation}'" ) if env.reward_space: reward_name = f"{reward_aggregation_name} {env.reward_space.id}" else: reward_name = "" # Determine the maximum column width required for printing tabular output. max_state_name_length = max( len(s) for s in [state_name(s) for s in states] + [ "Mean inference walltime", reward_name, ] ) name_col_width = min(max_state_name_length + 2, 78) error_count = 0 rewards = [] walltimes = [] if FLAGS.summary_only: def intermediate_print(*args, **kwargs): del args del kwargs else: intermediate_print = print def progress_message(i): intermediate_print( f"{i} remaining {plural(i, 'state', 'states')} to validate ... ", end="", flush=True, ) progress_message(len(states)) result_dicts = [] def dump_result_dicst_to_json(): with open(FLAGS.validation_logfile, "w") as f: json.dump(result_dicts, f) for i, result in enumerate(validation_results, start=1): intermediate_print("\r\033[K", to_string(result, name_col_width), sep="") progress_message(len(states) - i) result_dicts.append(result.dict()) if not result.okay(): error_count += 1 elif result.reward_validated and not result.reward_validation_failed: rewards.append(result.state.reward) walltimes.append(result.state.walltime) if not i % 10: dump_result_dicst_to_json() dump_result_dicst_to_json() # Print a summary footer. intermediate_print("\r\033[K----", "-" * name_col_width, "-----------", sep="") print(f"Number of validated results: {emph(len(walltimes))} of {len(states)}") walltime_mean = f"{arithmetic_mean(walltimes):.3f}s" walltime_std = f"{stdev(walltimes):.3f}s" print( f"Mean walltime per benchmark: {emph(walltime_mean)} " f"(std: {emph(walltime_std)})" ) reward = f"{reward_aggregation(rewards):.3f}" reward_std = f"{stdev(rewards):.3f}" print(f"{reward_name}: {emph(reward)} " f"(std: {emph(reward_std)})") if error_count: sys.exit(1)
def make_env(): return env_from_flags(benchmark=benchmark_from_flags())
def main(argv): """Main entry point.""" if len(argv) != 1: raise app.UsageError(f"Unknown command line arguments: {argv[1:]}") env = env_from_flags() try: if not env.datasets_site_path: raise app.UsageError("Environment has no benchmarks site path") env.datasets_site_path.mkdir(parents=True, exist_ok=True) env.inactive_datasets_site_path.mkdir(parents=True, exist_ok=True) invalidated_manifest = False for name_or_url in FLAGS.download: require(env, name_or_url) if FLAGS.download_all: for dataset in env.available_datasets: require(env, dataset) for name in FLAGS.activate: activate(env, name) invalidated_manifest = True if FLAGS.activate_all: for path in env.inactive_datasets_site_path.iterdir(): activate(env, path.name) invalidated_manifest = True for name in FLAGS.deactivate: deactivate(env, name) invalidated_manifest = True if FLAGS.deactivate_all: for path in env.datasets_site_path.iterdir(): deactivate(env, path.name) invalidated_manifest = True for name in FLAGS.delete: delete(env, name) if invalidated_manifest: env.make_manifest_file() print(f"{env.spec.id} benchmarks site dir: {env.datasets_site_path}") print() print(enumerate_directory("Active Datasets", env.datasets_site_path), ) print( "These benchmarks are ready for use. Deactivate them using `--deactivate=<name>`." ) print() print( enumerate_directory("Inactive Datasets", env.inactive_datasets_site_path)) print("These benchmarks may be activated using `--activate=<name>`.") print() print( tabulate( sorted([( d.name, d.license, humanize.intcomma(d.file_count), humanize.naturalsize(d.size_bytes), ) for d in env.available_datasets.values()]), headers=( "Downloadable Dataset", "License", "#. Benchmarks", "Size on disk", ), )) print( "These benchmarks may be installed using `--download=<name> --activate=<name>`." ) finally: env.close()
def main(argv): """Main entry point.""" assert len(argv) == 1, f"Unrecognized flags: {argv[1:]}" # Parse the input states from the user. states = list(read_states_from_stdin()) # Send the states off for validation validation_results = validate_states( env_from_flags, states, datasets=FLAGS.dataset, nproc=FLAGS.nproc, inorder=FLAGS.inorder, ) # Determine the name of the reward space. env = env_from_flags() try: if env.reward_space: gmean_name = f"Geometric mean {env.reward_space.id}" else: gmean_name = "Geometric mean" finally: env.close() # Determine the maximum column width required for printing tabular output. max_state_name_length = max( len(s) for s in [state_name(s) for s in states] + [ "Mean inference walltime", gmean_name, ]) name_col_width = min(max_state_name_length + 2, 78) error_count = 0 rewards = [] walltimes = [] for result in validation_results: print(to_string(result, name_col_width)) if result.failed: error_count += 1 elif result.reward_validated and not result.reward_validation_failed: rewards.append(result.state.reward) walltimes.append(result.state.walltime) # Print a summary footer. print("----", "-" * name_col_width, "-----------", sep="") print( f"Number of validated results: {emph(len(walltimes))} of {len(states)}" ) walltime_mean = f"{arithmetic_mean(walltimes):.3f}s" walltime_std = f"{stdev(walltimes):.3f}s" print(f"Mean inference walltime: {emph(walltime_mean)} sec / benchmark " f"(std: {emph(walltime_std)})") reward_gmean = f"{geometric_mean(rewards):.3f}" reward_std = f"{stdev(rewards):.3f}" print(f"{gmean_name}: {emph(reward_gmean)} (std: {emph(reward_std)})") if error_count: sys.exit(1)