class Game_Env():
def __init__(self, gameDisplay, game_matrix):
self.HEIGHT = game_matrix.ROWS #height of the environment
self.WIDTH = game_matrix.COLUMNS #width of the environment
self.DISPLAY = gameDisplay #will be used for rendering
display_width, display_height = gameDisplay.get_size()
display_height -= 100 #since we need some space to show important data.
self.BLOCK_WIDTH = int(display_width / self.WIDTH)
self.BLOCK_HEIGHT = int(display_height / self.HEIGHT)
#defining agents
self.CAT = Cat(self.DISPLAY, self.BLOCK_WIDTH, self.BLOCK_HEIGHT)
self.MOUSE = Mouse(self.DISPLAY, self.BLOCK_WIDTH, self.BLOCK_HEIGHT)
self.MOVES = {'mouse': 150, 'cat': 150}
self.OBSTACLES = game_matrix.OBSTACLES
#and finally the golden cheese
self.CHEESE_IMG = pygame.transform.scale(
pygame.image.load('pics/cheese.png'),
(self.BLOCK_WIDTH, self.BLOCK_HEIGHT))
#---------------------------------------------Returns the state of the environment----------------------------------#
def get_state(self):
#later on give cheese state to cat also
self.STATE = {
'mouse':
(self.MOUSE_X - self.CAT_X, self.MOUSE_Y - self.CAT_Y,
self.MOUSE_X - self.CHEESE_X, self.MOUSE_Y - self.CHEESE_Y),
'cat': (self.CAT_X - self.MOUSE_X, self.CAT_Y - self.MOUSE_Y)
}
return self.STATE
#--------------------------------------------Reset the environment--------------------------------------------------#
def reset(self):
self.MOUSE_X, self.MOUSE_Y = (0, 0)
self.CAT_X, self.CAT_Y = (0, self.HEIGHT - 1)
self.CHEESE_X, self.CHEESE_Y = np.random.randint(0, 9, 2, 'int')
#making sure cheese is not at obstacles
for obs in self.OBSTACLES:
if self.CHEESE_X == obs[0] and self.CHEESE_Y == obs[1]:
#then shift it up
self.CHEESE_Y -= 1
self.MOVES['cat'] = 100
self.MOVES['cat'] = 100
return self.get_state()
#-----------------------------------------------Render the enviroment-----------------------------------------------#
def render(self, i_episode=-1):
'''
rendering the environment using pygame display
'''
#drawing our agents
self.MOUSE.draw(self.MOUSE_X, self.MOUSE_Y)
self.CAT.draw(self.CAT_X, self.CAT_Y)
self.DISPLAY.blit(self.CHEESE_IMG, (self.CHEESE_X * self.BLOCK_WIDTH,
self.CHEESE_Y * self.BLOCK_HEIGHT))
#drawing obstacles
for pos in self.OBSTACLES:
pygame.draw.rect(self.DISPLAY, BLUE, [
pos[0] * self.BLOCK_WIDTH, pos[1] * self.BLOCK_HEIGHT,
self.BLOCK_WIDTH, self.BLOCK_HEIGHT
])
if i_episode >= 0:
self.display_episode(i_episode)
#--------------------------------------Agents takes step and the environment changes------------------------------------#
def step(self, mouse_action, cat_action):
reward = {'mouse': -1, 'cat': -1}
done = False
info = {
'cheese_eaten': False,
'mouse_caught': False,
'x': -1,
'y': -1,
'width': self.BLOCK_WIDTH,
'height': self.BLOCK_HEIGHT
}
#decreasing the no. of moves
self.MOVES['cat'] -= 1
self.MOVES['mouse'] -= 1
#done if moves = 0
if self.MOVES['cat'] == 0 or self.MOVES['mouse'] == 0:
done = True
self.update_positions(mouse_action, cat_action)
#mouse reached the cheese
if self.MOUSE_X == self.CHEESE_X and self.MOUSE_Y == self.CHEESE_Y:
done = True
reward['mouse'] = 50
info['cheese_eaten'], info['x'], info[
'y'] = True, self.MOUSE_X, self.MOUSE_Y
#cat caught the mouse
if self.CAT_X == self.MOUSE_X and self.CAT_Y == self.MOUSE_Y:
done = True
reward['cat'] = 50
reward['mouse'] = -20
info['mouse_caught'], info['x'], info[
'y'] = True, self.MOUSE_X, self.MOUSE_Y
for obs in self.OBSTACLES:
if self.MOUSE_X == obs[0] and self.MOUSE_Y == obs[1]:
reward['mouse'] = -20
self.MOUSE_X, self.MOUSE_Y = (0, 0)
if self.CAT_X == obs[0] and self.CAT_Y == obs[1]:
reward['cat'] = -20
self.CAT_X, self.CAT_Y = (0, self.HEIGHT - 1)
return self.get_state(), reward, done, info
#----------------------------Helper function to display episode-------------------------#
def display_episode(self, epsiode):
font = pygame.font.SysFont(None, 25)
text = font.render("Episode: " + str(epsiode), True, TEXT_COLOR)
self.DISPLAY.blit(text, (1, 1))
#-------------------------------Decide position changes based on action taken--------------#
def get_changes(self, action):
x_change, y_change = 0, 0
#decide action
if action == 0:
x_change = -1 #moving left
elif action == 1:
x_change = 1 #moving right
elif action == 2:
y_change = -1 #moving upwards
elif action == 3:
y_change = 1 #moving downwards
return x_change, y_change
#-----------------------------update positions of agents-------------------------------------#
def update_positions(self, mouse_action, cat_action):
x_change_mouse, y_change_mouse = self.get_changes(mouse_action)
x_change_cat, y_change_cat = self.get_changes(cat_action)
self.MOUSE_X += x_change_mouse
self.MOUSE_Y += y_change_mouse
self.CAT_X += x_change_cat
self.CAT_Y += y_change_cat
self.MOUSE_X, self.MOUSE_Y = self.fix(self.MOUSE_X, self.MOUSE_Y)
self.CAT_X, self.CAT_Y = self.fix(self.CAT_X, self.CAT_Y)
#------------------------------Push them back in to fight! There's no escape--------------------#
def fix(self, x, y):
# If agents out of bounds, fix!
if x < 0:
x = 0
elif x > self.WIDTH - 1:
x = self.WIDTH - 1
if y < 0:
y = 0
elif y > self.HEIGHT - 1:
y = self.HEIGHT - 1
return x, y
class AutoSimulation:
def __init__(self):
self.autos = []
self.events = []
self.generation = 0
self.dt = 0
self.top_auto = None
self.slow_car_removed = False
self.learning_rate = LEARNING_RATE
self.viewer = TargetViewer(self)
self.mouse = Mouse(self)
self.track = RaceTrack(self)
self.text = Text("Generation : 0")
self.start_time = pygame.time.get_ticks()
self.addNewAutos(num_car)
def generateTrack(self, difficulty):
size = 700 # make it fixed size, previous code was : #randint(int(800 - (400 * difficulty)), 800)
width = size // 4 # original is 4
self.track.generateTrack(
Rect((size, size), (-size / 2.0, -size / 2.0)), width)
def addNewAutos(self, n):
for _ in range(n):
self.autos.append(Auto(self))
def getFittestAuto(self):
max_fit = -1
top_autos = []
for auto in self.autos:
auto.top = 0
if auto.fitness > max_fit:
max_fit = auto.fitness
top_autos = [auto]
if not auto.stop:
pass
if auto.fitness == max_fit:
top_autos.append(auto)
if len(top_autos) == 1:
return top_autos[0]
gate_pos = top_autos[0].getCurrentGate().next_track.gate.getCenter()
closest = None
top_auto = None
for auto in top_autos:
dis = gate_pos.getDis(auto.pos)
if closest is None or dis < closest:
closest = dis
top_auto = auto
top_auto.top = 1
self.top_auto = top_auto
def start(self):
difficulty = 1.0 - (1.0 / ((self.generation / 100.0) + 1.0))
self.generateTrack(difficulty)
self.text = Text("Generation : " + str(self.generation))
self.start_time = pygame.time.get_ticks()
for car in self.autos:
car.start()
def update(self):
for car in self.autos:
if not car.stop: # only update unstopped car
car.update()
if (not self.slow_car_removed) and (
(pygame.time.get_ticks() - self.start_time) > 10000):
"""remove slow car after N ms"""
if car.vel.getMag() < 10:
car.stop = 1
self.getFittestAuto()
self.mouse.update()
#if not self.top_auto.stop:
# self.viewer.updatePos(self.top_auto.body.pos)
# self.viewer.updatePos(None) # KERU fix the view at the center
if pygame.time.get_ticks() - self.start_time > reset_timer:
self.reset()
def cont(self):
"""count stopped car and decide to continue the race"""
stopped = 0 # was 0, but 1 is a dumb hack to stop the race if only 1 is running
for car in self.autos: # count stopped car
stopped += car.stop
all_stopped = (stopped == (len(self.autos)))
return 1 - all_stopped
def end(self):
new_batch = []
# update the learning rate
if self.learning_rate > LEARNING_RATE_MIN:
self.learning_rate -= LEARNING_RATE_DEC
else:
self.learning_rate = LEARNING_RATE_MIN
print("Learning rate : %.4f" % self.learning_rate)
#add the top auto to next run
new_batch.append(self.top_auto)
# count running car, for display
running_car_count = 0
for running_auto in self.autos:
if running_auto.stop == 0:
running_car_count += 1
print("running cars : ", running_car_count)
# count car that are still running and send them for the next run
running_car_count = 0
for running_auto in self.autos:
if running_auto.stop == 0 and running_car_count < num_car // 2:
running_car_count += 1
## make slightly less mutated child
new_batch.append(running_auto.makeChild(self.learning_rate))
#complete the list up to 50% with mutated top car
new_top_car = 0
while running_car_count < (num_car // 2):
running_car_count += 1
new_top_car += 1
new_batch.append(self.top_auto.makeChild(self.learning_rate))
# complete the remaining with the list with mutated version of random car of this run
# leave one for top car
for i in range(len(self.autos) - running_car_count - 1):
#auto = self.top_auto.makeChild()
auto = choice(self.autos).makeChild(self.learning_rate)
new_batch.append(auto)
# finally, copy the new batch + the top auto (without mutating it) to the next batch
self.autos = new_batch + [self.top_auto]
self.generation += 1
print("Generation:", self.generation - 1, "Done!")
def render(self):
for car in self.autos:
if not car.stop: # only draw car that are still running
car.draw()
self.top_auto.draw()
self.top_auto.brain.draw(
Rect((300, 200), (self.viewer.pos.x + 15, self.viewer.pos.y + 5)))
self.track.draw()
self.mouse.draw()
self.text.draw(self.viewer.display)
self.viewer.draw(self.top_auto.inputs) #enable to see sensor
self.viewer.render()
self.viewer.clear()
def reset(self):
self.end()
self.start()
world = Map(Rect(0, 0, 600, 550))
world.load('mapdata.xml')
world.sprites = spriteset
world.start_fight = lambda h, m: draw_stack.open(win.CombatWindow(h, m))
gui.add(world)
sidebar = StatsBar(Rect(620, 20, 150, 300), spriteset)
world.tile_selected = sidebar.display
gui.add(sidebar)
game_over = False
while not game_over:
world.update()
current_gui = draw_stack.top
c = current_gui.get_cursor_at(mouse.get_pos())
# XXX draw all the objects here
display.fill((0, 0, 0))
current_gui.draw(display)
mouse.draw(display, c)
# overlays = pygame.sprite.RenderUpdates()
# overlays.draw(screen)
pygame.display.flip()
clock.tick(15)
for event in pygame.event.get():
if event.type == pygame.locals.QUIT:
game_over = True
current_gui.process_event(event)
class Game:
def __init__(self, total_episodes: int):
self.window_width = constant.WIDTH * constant.TILE
self.window_height = constant.HEIGHT * constant.TILE
self._running = True
self._display = None
self._snake = None
self._mouse = None
self.episode = 1
self.total_episodes = total_episodes
self.score = 0
self.max_score = 0
self.frames = 0
self.game_stats = []
self.specs = []
self.test_run = False
self.snake = Snake()
self.mouse = Mouse(constant.WIDTH, constant.HEIGHT,
self.snake.body_position())
self.q = QLearning()
def initialize_pygame(self):
"""
Initialize pygame along with display and image settings
"""
pygame.init()
self._display = pygame.display.set_mode(
(self.window_width, self.window_height), pygame.HWSURFACE)
pygame.display.set_caption('SNAKE ' + 'Episode ' + str(self.episode))
self._snake = pygame.image.load("img/snake_body_mini.png").convert()
# source for mouse: http://pixelartmaker.com/art/3d272b1bf180b60.png
self._mouse = pygame.image.load("img/mouse_mini.png").convert()
def game_over(self, collision_type: str):
"""
Print game results and exit the game
"""
collision_value = -1 # represents body collision
if collision_type == 'the wall':
collision_value = 1
self.snake.update_tail()
self._running = False
if self.score > self.max_score:
self.max_score = self.score
self.game_stats.append([self.frames, self.score, collision_value])
self.display(collision_type)
self.next_episode()
def display(self, collision_type: str):
"""
Displays game over status and scores, and can call display/save data functions
:param collision_type: what type of collision ended the game
"""
if self.episode % constant.SAVE_EPISODE == 0:
self.q.save_table(self.episode, clear_dir=constant.DELETE_JSON)
print(f'GAME OVER! Snake collided with {collision_type}')
print(f'SCORE: {self.score}')
def move_snake(self, ai_play: bool):
"""
Check whether the snake has eaten the mouse or encountered a collision
:param ai_play: True if ai play, False otherwise
"""
self.snake.update_head()
# if snake eats mouse
if self.snake.eats_mouse(self.mouse.x, self.mouse.y):
self.mouse.generate_mouse(self.snake.body_position())
self.score += 1
if ai_play:
self.q.update_reward('mouse')
# if snake collides with itself
elif self.snake.body_collision():
if ai_play:
self.q.update_reward('snake')
self.game_over('itself')
# if snake collides with walls
elif self.snake.wall_collision(0, self.window_width, 0,
self.window_height):
if ai_play:
self.q.update_reward('wall')
self.game_over('the wall')
else:
if ai_play:
self.q.update_reward('empty')
self.snake.update_tail()
def abs_coordinates(self):
snake_head = self.snake.head_coordinates()
mouse_loc = self.mouse.relative_coordinates(snake_head)
tail_loc = self.snake.tail_coordinates()
return tail_loc, mouse_loc
def render(self):
"""
Render the visual components of the game
"""
self._display.fill((0, 0, 0))
self.snake.draw(self._display, self._snake)
self.mouse.draw(self._display, self._mouse)
pygame.display.flip()
def human_play(self, delay: int):
"""
Executes the game play, snake movements, and loops until the game ends.
Keys can be used to play the game.
:param delay: defines the frame delay with lower values (e.g. 1) resulting in a fast frame, while higher values
(e.g. 1000) result in very slow frames
"""
while self._running:
pygame.event.pump()
keys = pygame.key.get_pressed()
if keys[pygame.K_RIGHT]:
self.snake.set_east()
elif keys[pygame.K_LEFT]:
self.snake.set_west()
elif keys[pygame.K_UP]:
self.snake.set_north()
elif keys[pygame.K_DOWN]:
self.snake.set_south()
elif keys[pygame.K_ESCAPE]:
self._running = False
self.move_snake(False)
self.render()
sleep(float(delay) / 1000)
self.frames += 1
def set_direction(self, direction: str):
"""
Sets the direction for the snake to take
:param direction: specified direction
"""
if direction == 'east':
self.snake.set_east()
elif direction == 'west':
self.snake.set_west()
elif direction == 'north':
self.snake.set_north()
else: # south
self.snake.set_south()
def ai_train(self, delay: int, resume_state: bool):
"""
Executes the AI training, looping until the snake is trained the total number of episodes.
Movements are implemented by the AI rather than by a human pressing keys.
:param delay: defines the frame delay with lower values (e.g. 1) resulting in a fast frame, while higher values
(e.g. 1000) result in very slow frames
:param resume_state: if True, start training from externally saved table's next episode, if False,
initial episode is 1
"""
# If resuming from a saved state, start from the loaded state's next episode
if resume_state:
self.resume_game(self.total_episodes)
while self._running:
pygame.event.pump()
tail_loc, mouse_loc = self.abs_coordinates()
snake_direction = self.snake.current_direction()
state = self.q.define_state(tail_loc, mouse_loc, snake_direction)
action = self.q.select_action(state)
self.set_direction(action)
self.move_snake(True)
tail_loc, mouse_loc = self.abs_coordinates()
snake_direction = self.snake.current_direction()
next_state = self.q.define_state(tail_loc, mouse_loc,
snake_direction)
self.q.update(state, next_state, action)
self.q.reset_reward()
self.render()
sleep(float(delay) / 1000)
self.frames += 1
def ai_test(self, delay: int, resume_state: bool):
"""
Tests the AI on previous training data
:param delay: defines the frame delay
:param resume_state: if True, start training from externally saved table's next episode, if False,
initial episode is 1
"""
self.test_run = True
self.episode = 1
# If resuming from a saved state, start from the loaded state's next episode
if resume_state:
self.resume_game(constant.TOTAL_TESTS)
if constant.PARAM_TEST:
self.total_episodes = constant.TOTAL_TESTS
# Run the total number of tests specified
while self.episode <= self.total_episodes:
caption = 'SNAKE ' + 'FINAL TEST RUN: EPISODE ' + str(self.episode)
self.reset_game(caption)
self.game_stats = []
self.specs = []
while self._running:
pygame.event.pump()
tail_loc, mouse_loc = self.abs_coordinates()
snake_direction = self.snake.current_direction()
state = self.q.define_state(tail_loc, mouse_loc,
snake_direction)
action = self.q.select_action(state)
self.set_direction(action)
self.move_snake(True)
self.render()
sleep(float(delay) / 1000)
self.frames += 1
print(
f'(TEST RUN EPISODE {str(self.episode)}) FINAL SCORE: {self.score}, FINAL MAX SCORE: {self.max_score}\n'
)
self.episode += 1
def resume_game(self, total_tests):
filename = 'episode' + str(constant.RESUME_EPISODE) + '.json'
self.episode = self.q.load_table(filename)
if self.episode < 1:
print(f'Table failed to load')
self.total_episodes = self.episode + total_tests - 1
def reset_game(self, caption: str):
pygame.display.set_caption(caption)
self.score = 0
self.frames = 0
self._running = True
self.snake.initialize_positions(self.mouse.x, self.mouse.y)
self.mouse.generate_mouse(self.snake.body_position())
def next_episode(self):
"""
Sets-up the next episode or completes the final episode
"""
if self.episode >= self.total_episodes:
self.prep_data()
return
# set new episode
self.episode += 1
print(f'\nNEW GAME, EPISODE {self.episode}')
caption = 'SNAKE ' + 'Episode ' + str(self.episode)
self.reset_game(caption)
def prep_data(self):
"""
Prepares data formatting with headers, specific test names, etc
"""
self.specs = []
filename = ''
if self.test_run:
filename = 'testing_' + constant.PARAM + str(constant.PARAM_VAL)
if constant.PARAM_TEST:
filename += constant.PARAM + str(constant.PARAM_VAL)
stats_file = filename + '_data.csv'
header = ['Steps', 'Scores', 'Collisions']
self.write_data(stats_file, header, self.game_stats)
specs_file = filename + '_specs.csv'
header = ['Parameters', 'Values']
self.specs.append(['total episodes', self.episode])
self.specs.append(['height', constant.HEIGHT])
self.specs.append(['width', constant.WIDTH])
self.specs.append(['learning rate', constant.ETA])
self.specs.append(['discount', constant.DISCOUNT])
self.specs.append(['epsilon', constant.EPSILON])
self.specs.append(['mouse reward', constant.MOUSE])
self.specs.append(['wall penalty', constant.WALL])
self.specs.append(['self-collision penalty', constant.SNAKE])
self.specs.append(['empty tile penalty', constant.EMPTY])
self.write_data(specs_file, header, self.specs, True)
def write_data(self,
filename: str,
header: [str],
data: [],
add_specs: bool = False):
"""
Writes the data from the current session to a file.
:param filename: filename to write data
:param header: header names for data
:param data: data to add to file
:param add_specs: True if writing specs file, False otherwise
"""
op = 'w' # default write to CSV
path = constant.DATA_DIR
file = path + filename
# create directory if it doesn't exist
if not os.path.exists(path):
os.mkdir(path)
# append data to existing file
if constant.RESUME and os.path.isfile(file):
op = 'a'
# write specs
if add_specs:
op = 'w'
# write data to csv file(s)
with open(file, op, newline='') as outfile:
w = csv.writer(outfile)
if not constant.RESUME:
w.writerow(header)
if not constant.PARAM_TEST and constant.RESUME:
w.writerow(header)
w.writerows(data)
outfile.close()
