def run(lay, lay_name, pac, ghosts, disp, n_games=1, name='games'):
"""Run a few games and return/output their statistics."""
starttime = time.time()
print('*** Running %s on' % name, lay_name, '%d time(s).' % n_games)
games = pacman.run_games(lay,
pac,
ghosts,
disp,
n_games,
False,
catch_exceptions=True,
timeout=120)
print('*** Finished running %s on' % name, lay_name,
'after %d seconds.' % (time.time() - starttime))
stats = {
'time': time.time() - starttime,
'wins': [g.state.is_win() for g in games].count(True),
'games': games,
'scores': [g.state.get_score() for g in games],
'timeouts': [g.agent_timeout for g in games].count(True),
'crashes': [g.agent_crashed for g in games].count(True)
}
print('*** Won %d out of %d games. Average score: %f ***' %
(stats['wins'], len(games), sum(stats['scores']) * 1.0 / len(games)))
return stats
def execute(self, grades, module_dict, solution_dict):
"""Test student's code.
Overrides test_classes.TestCase.execute
"""
start_time = time.time()
agent_type = getattr(module_dict['multi_agents'], self.agent_name)
agent_opts = (pacman.parse_agent_args(self.agent_args)
if self.agent_args != '' else {})
agent = agent_type(**agent_opts)
lay = layout.get_layout(self.layout_name, 3)
disp = self.question.display
random.seed(self.seed)
games = pacman.run_games(lay,
agent,
self.ghosts,
disp,
self.num_games,
False,
catch_exceptions=True,
timeout=self.max_time)
total_time = time.time() - start_time
stats = {
'time': total_time,
'wins': [g.state.is_win() for g in games].count(True),
'games': games,
'scores': [g.state.get_score() for g in games],
'timeouts': [g.agent_timeout for g in games].count(True),
'crashes': [g.agent_crashed for g in games].count(True)
}
average_score = sum(stats['scores']) / float(len(stats['scores']))
non_timeouts = self.num_games - stats['timeouts']
wins = stats['wins']
def grade_threshold(value, minimum, thresholds, name):
points = 0
passed = (minimum is None) or (value >= minimum)
if passed:
for t in thresholds:
if value >= t:
points += self.points_per_threshold
return (passed, points, value, minimum, thresholds, name)
results = [
grade_threshold(average_score, self.score_minimum,
self.score_thresholds, "average score"),
grade_threshold(non_timeouts, self.non_timeout_minimum,
self.non_timeout_thresholds,
"games not timed out"),
grade_threshold(wins, self.wins_minimum, self.wins_thresholds,
"wins")
]
total_points = 0
for passed, points, value, minimum, thresholds, name in results:
if minimum is None and len(thresholds) == 0:
continue
# print passed, points, value, minimum, thresholds, name
total_points += points
if not passed:
assert points == 0
self.add_message("%s %s (fail: below minimum value %s)" %
(value, name, minimum))
else:
self.add_message("%s %s (%s of %s points)" %
(value, name, points,
self.points_per_threshold * len(thresholds)))
if minimum is not None:
self.add_message(" Grading scheme:")
self.add_message(" < %-5s: fail" % (minimum, ))
if len(thresholds) == 0 or minimum != thresholds[0]:
self.add_message(" >= %-5s: 0 points" % (minimum, ))
for idx, threshold in enumerate(thresholds):
self.add_message(" >= %-5s: %s points" %
(threshold,
(idx + 1) * self.points_per_threshold))
elif len(thresholds) > 0:
self.add_message(" Grading scheme:")
self.add_message(" < %-5s: 0 points" % (thresholds[0], ))
for idx, threshold in enumerate(thresholds):
self.add_message(" >= %-5s: %s points" %
(threshold,
(idx + 1) * self.points_per_threshold))
if any([not passed for passed, _, _, _, _, _ in results]):
total_points = 0
return self.test_partial(grades, total_points, self.max_points)
def execute(self, grades, module_dict, solution_dict):
self.add_message('Grading agent using command: python pacman.py %s' %
(self.pacman_params, ))
start_time = time.time()
games = pacman.run_games(
**pacman.read_command(self.pacman_params.split(' ')))
total_time = time.time() - start_time
num_games = len(games)
stats = {
'time': total_time,
'wins': [g.state.is_win() for g in games].count(True),
'games': games,
'scores': [g.state.get_score() for g in games],
'timeouts': [g.agent_timeout for g in games].count(True),
'crashes': [g.agent_crashed for g in games].count(True)
}
average_score = sum(stats['scores']) / float(len(stats['scores']))
non_timeouts = num_games - stats['timeouts']
wins = stats['wins']
def grade_threshold(value, minimum, thresholds, name):
points = 0
passed = (minimum == None) or (value >= minimum)
if passed:
for t in thresholds:
if value >= t:
points += 1
return (passed, points, value, minimum, thresholds, name)
results = [
grade_threshold(average_score, self.score_minimum,
self.score_thresholds, "average score"),
grade_threshold(non_timeouts, self.non_timeout_minimum,
self.non_timeout_thresholds,
"games not timed out"),
grade_threshold(wins, self.wins_minimum, self.wins_thresholds,
"wins")
]
total_points = 0
for passed, points, value, minimum, thresholds, name in results:
if minimum == None and len(thresholds) == 0:
continue
# print passed, points, value, minimum, thresholds, name
total_points += points
if not passed:
assert points == 0
self.add_message("%s %s (fail: below minimum value %s)" %
(value, name, minimum))
else:
self.add_message("%s %s (%s of %s points)" %
(value, name, points, len(thresholds)))
if minimum != None:
self.add_message(" Grading scheme:")
self.add_message(" < %s: fail" % (minimum, ))
if len(thresholds) == 0 or minimum != thresholds[0]:
self.add_message(" >= %s: 0 points" % (minimum, ))
for idx, threshold in enumerate(thresholds):
self.add_message(" >= %s: %s points" %
(threshold, idx + 1))
elif len(thresholds) > 0:
self.add_message(" Grading scheme:")
self.add_message(" < %s: 0 points" % (thresholds[0], ))
for idx, threshold in enumerate(thresholds):
self.add_message(" >= %s: %s points" %
(threshold, idx + 1))
if any([not passed for passed, _, _, _, _, _ in results]):
total_points = 0
return self.test_partial(grades, total_points, self.max_points)