def evaluate_model(self):
logging.info("SimPLe epoch [% 6d]: evaluating model.",
self._simple_epoch)
start_time = time.time()
self._sim_env.initialize(
batch_size=self._simulated_batch_size,
history_stream=itertools.repeat(None),
)
if self._has_own_data:
trajectory_dir = self._trajectory_dump_root_dir
else:
trajectory_dir = self._initial_trajectory_dir
(_, eval_trajectories) = simple.load_trajectories(
trajectory_dir, eval_frac=self._data_eval_frac)
chosen_trajectories = [
random.choice(eval_trajectories)
for _ in range(self._sim_env.batch_size)
]
summaries = simple.evaluate_model(self._sim_env, chosen_trajectories,
plt)
if summaries is not None:
for (name, value) in summaries.items():
self._summary_writer.scalar("simple/{}".format(name),
value,
step=self._simple_epoch)
self._summary_writer.plot("simple/model_eval_plot",
plt,
step=self._simple_epoch)
self.flush_summaries()
logging.vlog(1, "Evaluating model took %0.2f sec.",
time.time() - start_time)
def make_example_streams(trajectory_dir):
(train_trajs, eval_trajs) = simple.load_trajectories(
trajectory_dir, eval_frac=self._data_eval_frac)
generate_examples = functools.partial(
simple.generate_examples,
trajectory_to_training_examples_fn=(
self._sim_env.trajectory_to_training_examples),
)
return tuple(map(generate_examples, (train_trajs, eval_trajs)))
def _load_trajectories(self, initial):
# Cache the initial trajectories in memory, as loading them can take a lot
# of time and they don't change.
if initial:
if self._initial_trajectories is not None:
return self._initial_trajectories
trajectory_dir = self._initial_trajectory_dir
else:
trajectory_dir = self._trajectory_dump_root_dir
trajectories = simple.load_trajectories(trajectory_dir,
self._data_eval_frac)
if initial:
self._initial_trajectories = trajectories
return trajectories
def test_loads_trajectories(self):
temp_dir = self.get_temp_dir()
# Dump two trajectory pickles with given observations.
self._dump_trajectory_pickle(observations=[0, 1, 2, 3],
path=os.path.join(temp_dir, "0.pkl"))
self._dump_trajectory_pickle(observations=[4, 5, 6, 7],
path=os.path.join(temp_dir, "1.pkl"))
(train_trajs, eval_trajs) = simple.load_trajectories(temp_dir,
eval_frac=0.25)
extract_obs = lambda t: t.last_time_step.observation
# The order of pickles is undefined, so we compare sets.
actual_train_obs = set(map(extract_obs, train_trajs))
actual_eval_obs = set(map(extract_obs, eval_trajs))
# First 3 trajectories from each pickle go to train, the last one to eval.
expected_train_obs = {0, 1, 2, 4, 5, 6}
expected_eval_obs = {3, 7}
self.assertEqual(actual_train_obs, expected_train_obs)
self.assertEqual(actual_eval_obs, expected_eval_obs)