def test_ls(start_ray, tmpdir):
"""This test captures output of list_trials."""
experiment_name = "test_ls"
experiment_path = os.path.join(str(tmpdir), experiment_name)
num_samples = 2
tune.run_experiments({
experiment_name: {
"run": "__fake",
"stop": {
"training_iteration": 1
},
"num_samples": num_samples,
"local_dir": str(tmpdir)
}
})
with Capturing() as output:
commands.list_trials(experiment_path, info_keys=("status", ))
lines = output.captured
assert sum("TERMINATED" in line for line in lines) == num_samples
with Capturing() as output:
commands.list_trials(experiment_path,
info_keys=("status", ),
filter_op="status == TERMINATED")
lines = output.captured
assert sum("TERMINATED" in line for line in lines) == num_samples
def test_ls(start_ray, tmpdir):
"""This test captures output of list_trials."""
experiment_name = "test_ls"
experiment_path = os.path.join(str(tmpdir), experiment_name)
num_samples = 3
tune.run("__fake",
name=experiment_name,
stop={"training_iteration": 1},
num_samples=num_samples,
local_dir=str(tmpdir),
global_checkpoint_period=0)
columns = ["status", "episode_reward_mean", "training_iteration"]
limit = 2
with Capturing() as output:
commands.list_trials(experiment_path, info_keys=columns, limit=limit)
lines = output.captured
assert all(col in lines[1] for col in columns)
assert lines[1].count("|") == len(columns) + 1
assert len(lines) == 3 + limit + 1
with Capturing() as output:
commands.list_trials(experiment_path,
sort=["status"],
info_keys=("status", ),
filter_op="status == TERMINATED")
lines = output.captured
assert sum("TERMINATED" in line for line in lines) == num_samples
assert len(lines) == 3 + num_samples + 1
def list_trials(experiment_path, sort, output, filter_op, columns,
result_columns):
"""Lists trials in the directory subtree starting at the given path."""
if columns:
columns = columns.split(",")
if result_columns:
result_columns = result_columns.split(",")
commands.list_trials(experiment_path, sort, output, filter_op, columns,
result_columns)
def list_trials(experiment_path, sort, output, filter_op, columns, limit,
desc):
"""Lists trials in the directory subtree starting at the given path."""
if sort:
sort = sort.split(",")
if columns:
columns = columns.split(",")
commands.list_trials(experiment_path, sort, output, filter_op, columns,
limit, desc)
def test_ls_with_cfg(start_ray, tmpdir):
experiment_name = "test_ls_with_cfg"
experiment_path = os.path.join(str(tmpdir), experiment_name)
tune.run("__fake",
name=experiment_name,
stop={"training_iteration": 1},
config={"test_variable": tune.grid_search(list(range(5)))},
local_dir=str(tmpdir))
columns = [CONFIG_PREFIX + "test_variable", "trial_id"]
limit = 4
with Capturing() as output:
commands.list_trials(experiment_path, info_keys=columns, limit=limit)
lines = output.captured
assert all(col in lines[1] for col in columns)
assert lines[1].count("|") == len(columns) + 1
assert len(lines) == 3 + limit + 1
def test_ls(start_ray, capsys, tmpdir):
"""This test captures output of list_trials."""
experiment_name = "test_ls"
experiment_path = os.path.join(str(tmpdir), experiment_name)
num_samples = 2
with capsys.disabled():
tune.run_experiments({
experiment_name: {
"run": "__fake",
"stop": {
"training_iteration": 1
},
"num_samples": num_samples,
"local_dir": str(tmpdir)
}
})
commands.list_trials(experiment_path, info_keys=("status", ))
captured = capsys.readouterr().out.strip()
lines = captured.split("\n")
assert sum("TERMINATED" in line for line in lines) == num_samples
def test_ls(start_ray, tmpdir):
"""This test captures output of list_trials."""
experiment_name = "test_ls"
experiment_path = os.path.join(str(tmpdir), experiment_name)
num_samples = 3
tune.run(
"__fake",
name=experiment_name,
stop={"training_iteration": 1},
num_samples=num_samples,
local_dir=str(tmpdir),
)
columns = ["episode_reward_mean", "training_iteration", "trial_id"]
limit = 2
with Capturing() as output:
commands.list_trials(experiment_path, info_keys=columns, limit=limit)
lines = output.captured
assert all(col in lines[1] for col in columns)
assert lines[1].count("|") == len(columns) + 1
assert len(lines) == 3 + limit + 1
with Capturing() as output:
commands.list_trials(
experiment_path,
sort=["trial_id"],
info_keys=("trial_id", "training_iteration"),
filter_op="training_iteration == 1",
)
lines = output.captured
assert len(lines) == 3 + num_samples + 1
with pytest.raises(click.ClickException):
commands.list_trials(experiment_path,
sort=["trial_id"],
info_keys=("training_iteration", ))
with pytest.raises(click.ClickException):
commands.list_trials(experiment_path, info_keys=("asdf", ))
def list_trials(experiment_path, sort, output):
"""Lists trials in the directory subtree starting at the given path."""
commands.list_trials(experiment_path, sort, output)
def list_trials(experiment_path, sort, output):
"""Lists trials in the directory subtree starting at the given path."""
commands.list_trials(experiment_path, sort, output)
def _export_tune_experiment(self, export_directory, output_file="out.csv"):
tmpkeys = commands.DEFAULT_CLI_KEYS
commands.DEFAULT_CLI_KEYS = AllSet()
commands.list_trials(path.join(export_directory, self.experiment_name), output=path.join(export_directory, output_file))
commands.DEFAULT_CLI_KEYS = tmpkeys
