def print_service_capabilities(env: CompilerEnv):
"""Discover and print the capabilities of a CompilerGym service.
:param env: An environment.
"""
print(header("Datasets"))
print(summarize_datasets(env.datasets), )
print(header("Observation Spaces"))
print(
tabulate(
sorted([(space, f"`{truncate(shape.space, max_line_len=80)}`")
for space, shape in env.observation.spaces.items()]),
headers=("Observation space", "Shape"),
))
print(header("Reward Spaces"))
print(
tabulate(
[(
name,
space.range,
space.success_threshold,
"Yes" if space.deterministic else "No",
"Yes" if space.platform_dependent else "No",
) for name, space in sorted(env.reward.spaces.items())],
headers=(
"Reward space",
"Range",
"Success threshold",
"Deterministic?",
"Platform dependent?",
),
))
for action_space in env.action_spaces:
print(header(f"{action_space.name} Action Space"))
# Special handling for commandline action spaces to print additional
# information.
if isinstance(action_space, Commandline):
table = tabulate(
[(f"`{n}`", d) for n, d in zip(
action_space.names,
action_space.descriptions,
)],
headers=("Action", "Description"),
)
print(table)
elif isinstance(action_space, NamedDiscrete):
table = tabulate(
[(a, ) for a in sorted(action_space.names)],
headers=("Action", ),
)
print(table)
else:
print(f'Action space "{action_space}" is not supported.')
def print_service_capabilities(env: CompilerEnv, base_heading_level: int = 1):
"""Discover and print the capabilities of a CompilerGym service.
:param env: An environment.
"""
print(
header(f"CompilerGym Service `{env.service}`",
base_heading_level).strip())
print(header("Programs", base_heading_level + 1))
print(
tabulate(
[(p, ) for p in sorted(env.benchmarks)],
headers=("Benchmark", ),
))
print(header("Observation Spaces", base_heading_level + 1))
print(
tabulate(
sorted([(space, f"`{shape}`")
for space, shape in env.observation.spaces.items()]),
headers=("Observation space", "Shape"),
))
print(header("Reward Spaces", base_heading_level + 1))
print(
tabulate(
sorted(env.reward.ranges.items()),
headers=("Reward space", "Range"),
))
print(header("Action Spaces", base_heading_level + 1).rstrip())
for action_space in env.action_spaces:
print(
header(
f"`{action_space.name}` ({type(action_space).__name__})",
base_heading_level + 2,
))
# Special handling for commandline action spaces to print additional
# information.
if isinstance(action_space, Commandline):
table = tabulate(
[(n, f"`{f}`", d) for n, f, d in zip(
action_space.names,
action_space.flags,
action_space.descriptions,
)],
headers=("Action", "Flag", "Description"),
)
else:
table = tabulate(
[(a, ) for a in sorted(action_space.names)],
headers=("Action", ),
)
print(table)
def enumerate_directory(name: str, path: Path):
rows = []
for path in path.iterdir():
if not path.is_file() or not path.name.endswith(".json"):
continue
dataset = Dataset.from_json_file(path)
rows.append((dataset.name, dataset.license, dataset.file_count,
dataset.size_bytes))
rows.append(
("Total", "", sum(r[2] for r in rows), sum(r[3] for r in rows)))
return tabulate(
[(n, l, humanize.intcomma(f), humanize.naturalsize(s))
for n, l, f, s in rows],
headers=(name, "License", "#. Benchmarks", "Size on disk"),
)
def replay_actions(env: CompilerEnv, action_names: List[str], outdir: Path):
logs_path = outdir / logs.BEST_ACTIONS_PROGRESS_NAME
start_time = time()
if config.enable_llvm_env:
if isinstance(env, LlvmEnv):
env.write_bitcode(outdir / "unoptimized.bc")
with open(str(logs_path), "w") as f:
ep_reward = 0
for i, action in enumerate(action_names, start=1):
_, reward, done, _ = env.step(env.action_space.names.index(action))
assert not done
ep_reward += reward
print(
f"Step [{i:03d} / {len(action_names):03d}]: reward={reward:.4f} \t"
f"episode={ep_reward:.4f} \taction={action}")
progress = RandomSearchProgressLogEntry(
runtime_seconds=time() - start_time,
total_episode_count=1,
total_step_count=i,
num_passes=i,
reward=reward,
)
print(progress.to_csv(), action, file=f, sep=",")
if config.enable_llvm_env:
if isinstance(env, LlvmEnv):
env.write_bitcode(outdir / "optimized.bc")
print(
tabulate(
[
(
"IR instruction count",
env.observation["IrInstructionCountO0"],
env.observation["IrInstructionCountOz"],
env.observation["IrInstructionCount"],
),
(
"Object .text size (bytes)",
env.observation["ObjectTextSizeO0"],
env.observation["ObjectTextSizeOz"],
env.observation["ObjectTextSizeBytes"],
),
],
headers=("", "-O0", "-Oz", "final"),
))
def summarize_datasets(datasets: Iterable[Dataset]) -> str:
rows = []
# Override the default iteration order of datasets.
for dataset in sorted(datasets, key=lambda d: d.name):
# Raw numeric values here, formatted below.
description = truncate(dataset.description, max_line_len=60)
links = ", ".join(f"`{name} <{url}>`__"
for name, url in sorted(dataset.references.items()))
if links:
description = f"{description} [{links}]"
rows.append((
dataset.name,
dataset.size,
description,
dataset.validatable,
))
rows.append(("Total", sum(r[1] for r in rows), "", ""))
return (tabulate(
[
(
n,
# A size of zero means infinite.
f"{f:,d}" if f > 0 else "∞",
l,
v,
) for n, f, l, v in rows
],
headers=(
"Dataset",
"Num. Benchmarks [#f1]_",
"Description",
"Validatable [#f2]_",
),
) + f"""
.. [#f1] Values obtained on {sys.platform}. Datasets are platform-specific.
.. [#f2] A **validatable** dataset is one where the behavior of the benchmarks
can be checked by compiling the programs to binaries and executing
them. If the benchmarks crash, or are found to have different behavior,
then validation fails. This type of validation is used to check that
the compiler has not broken the semantics of the program.
See :mod:`compiler_gym.bin.validate`.
""")
def do_stack(self, arg):
"""Show the environments on the stack. The current environment is the first shown."""
rows = []
total = 0
for i, hist in enumerate(self.stack):
name = hist.action_name
effect = hist.has_effect()
done = hist.done
reward = f"{hist.reward:.6f}" if hist.reward is not None else "-"
total += hist.reward or 0
row = (i + 1, name, effect, done, reward, f"{total:.6f}")
rows.append(row)
rows.reverse()
rows.append((0, "<init>", False, False, 0, 0))
headers = [
"Depth", "Action", "Effect", "Done", "Reward", "Cumulative Reward"
]
print(tabulate(rows, headers=headers, tablefmt="presto"))
def do_try_all_actions(self, args):
"""Tries all actions from this position and reports the results in sorted order by reward"""
if not self.env.reward_space:
print("No default reward set. Call set_default_reward")
return
with Timer("Got actions"):
items = self.get_action_rewards()
def row(item):
return (
item.action_name,
item.has_effect(),
item.done,
f"{item.reward:.6f}",
)
rows = [row(item) for item in items]
headers = ["Action", "Effect", "Done", "Reward"]
print(tabulate(rows, headers=headers, tablefmt="presto"))
def run_sensitivity_analysis_eval(rewards_path: Path,
runtimes_path: Path) -> None:
"""Print a summary of sensitivity analysis logs."""
with open(str(rewards_path)) as f:
rewards_in = f.read().rstrip().split("\n")
with open(str(runtimes_path)) as f:
runtimes_in = f.read().rstrip().split("\n")
rows = []
for rewards_row, runtimes_row in zip(rewards_in, runtimes_in):
name, *rewards = rewards_row.split(",")
_, *runtimes = runtimes_row.split(",")
if rewards == [""]:
rows.append((name, "-", "-", "-", "-", "-"))
continue
rewards = np.array([float(v) for v in rewards])
runtimes = np.array([float(v) for v in runtimes])
rows.append((
name,
humanize_duration(runtimes.mean()),
f"{rewards.mean():.5%}",
f"{np.median(rewards):.5%}",
f"{rewards.max():.5%}",
f"{rewards.std():.5%}",
))
print(
tabulate(
sorted(rows),
headers=(
"Name",
"Time (avg)",
"Δ (avg)",
"Δ (median)",
"Δ (max)",
"Δ (std.)",
),
))
def print_service_capabilities(env: CompilerEnv, base_heading_level: int = 1):
"""Discover and print the capabilities of a CompilerGym service.
:param env: An environment.
"""
print(header(f"CompilerGym Service `{env.service}`", base_heading_level).strip())
print(header("Programs", base_heading_level + 1))
print(
tabulate(
[(p,) for p in sorted(env.benchmarks)],
headers=("Benchmark",),
)
)
print(header("Observation Spaces", base_heading_level + 1))
print(
tabulate(
sorted(
[
(space, shape2str(shape))
for space, shape in env.observation.spaces.items()
]
),
headers=("Observation space", "Shape"),
)
)
print(header("Reward Spaces", base_heading_level + 1))
print(
tabulate(
[
(
name,
space.range,
space.success_threshold,
"Yes" if space.deterministic else "No",
"Yes" if space.platform_dependent else "No",
)
for name, space in sorted(env.reward.spaces.items())
],
headers=(
"Reward space",
"Range",
"Success threshold",
"Deterministic?",
"Platform dependent?",
),
)
)
print(header("Action Spaces", base_heading_level + 1).rstrip())
for action_space in env.action_spaces:
print(
header(
f"`{action_space.name}` ({type(action_space).__name__})",
base_heading_level + 2,
)
)
# Special handling for commandline action spaces to print additional
# information.
if isinstance(action_space, Commandline):
table = tabulate(
[
(f"`{n}`", d)
for n, d in zip(
action_space.names,
action_space.descriptions,
)
],
headers=("Action", "Description"),
)
else:
table = tabulate(
[(a,) for a in sorted(action_space.names)],
headers=("Action",),
)
print(table)
def eval_logs(outdir: Path) -> None:
rows = []
totals = {
"instructions": 0,
"init_reward": [],
"max_reward": [],
"attempts": 0,
"time": 0,
"actions": 0,
}
for results_dir in sorted(outdir.iterdir()):
benchmark = results_dir.name
progress_path = results_dir / logs.PROGRESS_LOG_NAME
meta_path = results_dir / logs.METADATA_NAME
if (not results_dir.is_dir() or not progress_path.is_file()
or not meta_path.is_file()):
continue
with open(meta_path, "rb") as f:
meta = json.load(f)
with open(str(progress_path)) as f:
final_line = f.readlines()[-1]
best = logs.ProgressLogEntry.from_csv(final_line)
totals["instructions"] += meta["num_instructions"]
totals["init_reward"].append(meta["init_reward"])
totals["max_reward"].append(best.reward)
totals["attempts"] += best.total_episode_count
totals["time"] += best.runtime_seconds
totals["actions"] += best.num_passes
rows.append((
benchmark,
humanize.intcomma(meta["num_instructions"]),
f"{meta['init_reward']:.4f}",
f"{best.reward:.4f}",
(f"{humanize.intcomma(best.total_episode_count)} attempts "
f"in {humanize.naturaldelta(best.runtime_seconds)}"),
humanize.intcomma(best.num_passes),
))
row_count = len(totals["init_reward"])
rows.append((
"Geomean",
"",
f"{geometric_mean(totals['init_reward']):.4f}",
f"{geometric_mean(totals['max_reward']):.4f}",
"",
"",
))
rows.append((
"Average",
humanize.intcomma(int(totals["instructions"] / row_count)),
f"{np.array(totals['init_reward']).mean():.4f}",
f"{np.array(totals['max_reward']).mean():.4f}",
(f"{humanize.intcomma(int(totals['attempts'] / row_count))} attempts "
f"in {humanize.naturaldelta(totals['time'] / row_count)}"),
humanize.intcomma(int(totals["actions"] / row_count)),
))
print(
tabulate(
rows,
headers=(
"Benchmark",
"#. instructions",
"Init Reward",
"Max Reward",
"Found after",
"#. actions",
),
))
def main(argv):
"""Main entry point."""
if len(argv) != 1:
raise app.UsageError(f"Unknown command line arguments: {argv[1:]}")
env = env_from_flags()
try:
if not env.datasets_site_path:
raise app.UsageError("Environment has no benchmarks site path")
env.datasets_site_path.mkdir(parents=True, exist_ok=True)
env.inactive_datasets_site_path.mkdir(parents=True, exist_ok=True)
invalidated_manifest = False
for name_or_url in FLAGS.download:
require(env, name_or_url)
if FLAGS.download_all:
for dataset in env.available_datasets:
require(env, dataset)
for name in FLAGS.activate:
activate(env, name)
invalidated_manifest = True
if FLAGS.activate_all:
for path in env.inactive_datasets_site_path.iterdir():
activate(env, path.name)
invalidated_manifest = True
for name in FLAGS.deactivate:
deactivate(env, name)
invalidated_manifest = True
if FLAGS.deactivate_all:
for path in env.datasets_site_path.iterdir():
deactivate(env, path.name)
invalidated_manifest = True
for name in FLAGS.delete:
delete(env, name)
if invalidated_manifest:
env.make_manifest_file()
print(f"{env.spec.id} benchmarks site dir: {env.datasets_site_path}")
print()
print(enumerate_directory("Active Datasets", env.datasets_site_path), )
print(
"These benchmarks are ready for use. Deactivate them using `--deactivate=<name>`."
)
print()
print(
enumerate_directory("Inactive Datasets",
env.inactive_datasets_site_path))
print("These benchmarks may be activated using `--activate=<name>`.")
print()
print(
tabulate(
sorted([(
d.name,
d.license,
humanize.intcomma(d.file_count),
humanize.naturalsize(d.size_bytes),
) for d in env.available_datasets.values()]),
headers=(
"Downloadable Dataset",
"License",
"#. Benchmarks",
"Size on disk",
),
))
print(
"These benchmarks may be installed using `--download=<name> --activate=<name>`."
)
finally:
env.close()
