def test_clip_average_benchmark_empty():
scores = scoring.benchmark_aggregate_score(benchmark, {})
benchmark_results = defaultdict(list)
task = benchmark.tasks[0]
env_id = task.env_id
benchmark_results[env_id].append(
_benchmark_result_helper(benchmark, env_id=env_id))
scores = scoring.benchmark_aggregate_score(benchmark, benchmark_results)
_assert_benchmark_score(scores,
score=0.00005,
num_envs_solved=0,
summed_training_seconds=1.0,
start_to_finish_seconds=1.0)
def test_reward_per_time_benchmark_empty():
scores = scoring.benchmark_aggregate_score(reward_per_time_benchmark, {})
benchmark_results = defaultdict(list)
task = reward_per_time_benchmark.tasks[0]
env_id = task.env_id
benchmark_results[env_id].append(
_benchmark_result_helper(reward_per_time_benchmark,
env_id=env_id,
episode_lengths=[10]))
scores = scoring.benchmark_aggregate_score(reward_per_time_benchmark,
benchmark_results)
_assert_benchmark_score(scores,
score=0.0,
num_envs_solved=0,
summed_training_seconds=0.0,
start_to_finish_seconds=0.0)
def test_clip_average_benchmark_incomplete():
benchmark_results = defaultdict(list)
env_id = benchmark.tasks[0].env_id
benchmark_results[env_id].append(
_benchmark_result_helper(benchmark, env_id=env_id, timestamps=[2]))
scores = scoring.benchmark_aggregate_score(benchmark, benchmark_results)
_assert_benchmark_score(scores,
score=0.00005,
num_envs_solved=0,
summed_training_seconds=1.0,
start_to_finish_seconds=1.0)
def test_clip_average_benchmark_scoring():
benchmark_results = defaultdict(list)
for i, task in enumerate(benchmark.tasks):
env_id = task.env_id
benchmark_results[env_id].append(
_benchmark_result_helper(benchmark,
env_id=env_id,
timestamps=[i + 2]))
scores = scoring.benchmark_aggregate_score(benchmark, benchmark_results)
_assert_benchmark_score(scores,
score=0.0001,
num_envs_solved=0,
summed_training_seconds=3.0,
start_to_finish_seconds=2.0)
def test_benchmark_simple():
# TODO what is this testing?
benchmark_results = defaultdict(list)
for i, task in enumerate(reward_benchmark.tasks):
env_id = task.env_id
benchmark_results[env_id].append(
_benchmark_result_helper(reward_benchmark,
env_id=env_id,
timestamps=[i + 2]))
scores = scoring.benchmark_aggregate_score(reward_benchmark,
benchmark_results)
_assert_benchmark_score(scores,
score=0.01,
num_envs_solved=0,
summed_training_seconds=3.0,
start_to_finish_seconds=2.0)
def test_clip_average_benchmark_solved():
benchmark_results = defaultdict(list)
N = 200
for i, task in enumerate(benchmark.tasks):
env_id = task.env_id
benchmark_results[env_id].append(
benchmark.score_evaluation(
env_id,
data_sources=[0] * N,
initial_reset_timestamps=[1],
episode_lengths=[1] * N,
episode_rewards=[1000] * N,
episode_types=['t'] * N,
timestamps=list(range(N)),
))
scores = scoring.benchmark_aggregate_score(benchmark, benchmark_results)
_assert_benchmark_score(scores,
score=1.0,
num_envs_solved=len(benchmark.tasks))
def test_reward_per_time_benchmark_solved():
benchmark_results = defaultdict(list)
N = 200
for i, task in enumerate(reward_per_time_benchmark.tasks):
env_id = task.env_id
benchmark_results[env_id].append(
reward_per_time_benchmark.score_evaluation(
env_id,
data_sources=[0] * N,
initial_reset_timestamps=[1],
episode_lengths=[1] * N,
episode_rewards=[1000] * N,
episode_types=['t'] * N,
timestamps=list(range(N)),
))
scores = scoring.benchmark_aggregate_score(reward_per_time_benchmark,
benchmark_results)
# Currently reward per time has no solved functionality, so num_envs_solved
# is 0
_assert_benchmark_score(scores, score=1.0, num_envs_solved=0)
def test_clip_average_benchmark_eval_handling():
# make sure we handle separate evaluation, training episodes properly
benchmark_results = defaultdict(list)
for i, task in enumerate(benchmark.tasks):
env_id = task.env_id
benchmark_results[env_id].append(
benchmark.score_evaluation(
env_id,
data_sources=[0, 1, 1],
initial_reset_timestamps=[1, 1],
episode_lengths=[1, 1, 1],
episode_rewards=[1, 2, 3],
episode_types=['e', 't', 'e'],
timestamps=[i + 2, i + 3, i + 4],
))
scores = scoring.benchmark_aggregate_score(benchmark, benchmark_results)
_assert_benchmark_score(scores,
score=0.0004,
num_envs_solved=0,
summed_training_seconds=5.0,
summed_task_wall_time=5.0,
start_to_finish_seconds=3.0)
def test_benchmark_extra():
benchmark_results = defaultdict(list)
for i, task in enumerate(reward_benchmark.tasks):
env_id = task.env_id
benchmark_results[env_id].append(
_benchmark_result_helper(reward_benchmark,
env_id=env_id,
timestamps=[i + 2]))
# add one more at the end with a high reward
benchmark_results[env_id].append(
_benchmark_result_helper(reward_benchmark,
env_id=env_id,
episode_rewards=[100],
timestamps=[2]))
scores = scoring.benchmark_aggregate_score(reward_benchmark,
benchmark_results)
_assert_benchmark_score(scores,
score=0.01,
num_envs_solved=0,
summed_training_seconds=3.0,
start_to_finish_seconds=2.0)
