def test_video_callable_records_videos():
with helpers.tempdir() as temp:
env = gym.make('CartPole-v0')
env = Monitor(env, temp)
env.reset()
env.close()
results = monitoring.load_results(temp)
assert len(results['videos']) == 1, "Videos: {}".format(results['videos'])
def test_video_callable_false_does_not_record():
with helpers.tempdir() as temp:
env = gym.make('CartPole-v0')
env = Monitor(env, temp, video_callable=False)
env.reset()
env.close()
results = monitoring.load_results(temp)
assert len(results['videos']) == 0
def upload_training_data(training_dir, api_key=None):
# Could have multiple manifests
results = monitoring.load_results(training_dir)
if not results:
raise error.Error('''Could not find any manifest files in {}.
(HINT: this usually means you did not yet close() your env.monitor and have not yet exited the process. You should call 'env.monitor.start(training_dir)' at the start of training and 'env.close()' at the end, or exit the process.)'''
.format(training_dir))
manifests = results['manifests']
env_info = results['env_info']
data_sources = results['data_sources']
timestamps = results['timestamps']
episode_lengths = results['episode_lengths']
episode_rewards = results['episode_rewards']
episode_types = results['episode_types']
initial_reset_timestamps = results['initial_reset_timestamps']
videos = results['videos']
env_id = env_info['env_id']
logger.debug('[%s] Uploading data from manifest %s', env_id,
', '.join(manifests))
# Do the relevant uploads
if len(episode_lengths) > 0:
training_episode_batch = upload_training_episode_batch(
data_sources,
episode_lengths,
episode_rewards,
episode_types,
initial_reset_timestamps,
timestamps,
api_key,
env_id=env_id)
else:
training_episode_batch = None
if len(videos) > MAX_VIDEOS:
logger.warning(
'[%s] You recorded videos for %s episodes, but the scoreboard only supports up to %s. We will automatically subsample for you, but you also might wish to adjust your video recording rate.',
env_id, len(videos), MAX_VIDEOS)
subsample_inds = np.linspace(0,
len(videos) - 1,
MAX_VIDEOS).astype('int') #pylint: disable=E1101
videos = [videos[i] for i in subsample_inds]
if len(videos) > 0:
training_video = upload_training_video(videos, api_key, env_id=env_id)
else:
training_video = None
return env_info, training_episode_batch, training_video
def test():
benchmark = registration.Benchmark(id='MyBenchmark-v0',
scorer=scoring.ClipTo01ThenAverage(),
tasks=[{
'env_id': 'CartPole-v0',
'trials': 1,
'max_timesteps': 5
}, {
'env_id': 'CartPole-v0',
'trials': 1,
'max_timesteps': 100,
}])
with helpers.tempdir() as temp:
env = gym.make('CartPole-v0')
env = wrappers.Monitor(env, directory=temp, video_callable=False)
env.seed(0)
env.set_monitor_mode('evaluation')
rollout(env)
env.set_monitor_mode('training')
for i in range(2):
rollout(env)
env.set_monitor_mode('evaluation')
rollout(env, good=True)
env.close()
results = monitoring.load_results(temp)
evaluation_score = benchmark.score_evaluation(
'CartPole-v0', results['data_sources'],
results['initial_reset_timestamps'], results['episode_lengths'],
results['episode_rewards'], results['episode_types'],
results['timestamps'])
benchmark_score = benchmark.score_benchmark({
'CartPole-v0':
evaluation_score['scores'],
})
assert np.all(
np.isclose(evaluation_score['scores'],
[0.00089999999999999998, 0.0054000000000000003
])), "evaluation_score={}".format(evaluation_score)
assert np.isclose(
benchmark_score,
0.00315), "benchmark_score={}".format(benchmark_score)
def test_only_complete_episodes_written():
with helpers.tempdir() as temp:
env = gym.make('CartPole-v0')
env = Monitor(env, temp, video_callable=False)
env.reset()
d = False
while not d:
_, _, d, _ = env.step(env.action_space.sample())
env.reset()
env.step(env.action_space.sample())
env.close()
# Only 1 episode should be written
results = monitoring.load_results(temp)
assert len(results['episode_lengths']) == 1, "Found {} episodes written; expecting 1".format(len(results['episode_lengths']))