class Game_Env(): def __init__(self, gameDisplay, game_matrix): self.HEIGHT = game_matrix.ROWS #height of the environment self.WIDTH = game_matrix.COLUMNS #width of the environment self.DISPLAY = gameDisplay #will be used for rendering display_width, display_height = gameDisplay.get_size() display_height -= 100 #since we need some space to show important data. self.BLOCK_WIDTH = int(display_width / self.WIDTH) self.BLOCK_HEIGHT = int(display_height / self.HEIGHT) #defining agents self.CAT = Cat(self.DISPLAY, self.BLOCK_WIDTH, self.BLOCK_HEIGHT) self.MOUSE = Mouse(self.DISPLAY, self.BLOCK_WIDTH, self.BLOCK_HEIGHT) self.MOVES = {'mouse': 150, 'cat': 150} self.OBSTACLES = game_matrix.OBSTACLES #and finally the golden cheese self.CHEESE_IMG = pygame.transform.scale( pygame.image.load('pics/cheese.png'), (self.BLOCK_WIDTH, self.BLOCK_HEIGHT)) #---------------------------------------------Returns the state of the environment----------------------------------# def get_state(self): #later on give cheese state to cat also self.STATE = { 'mouse': (self.MOUSE_X - self.CAT_X, self.MOUSE_Y - self.CAT_Y, self.MOUSE_X - self.CHEESE_X, self.MOUSE_Y - self.CHEESE_Y), 'cat': (self.CAT_X - self.MOUSE_X, self.CAT_Y - self.MOUSE_Y) } return self.STATE #--------------------------------------------Reset the environment--------------------------------------------------# def reset(self): self.MOUSE_X, self.MOUSE_Y = (0, 0) self.CAT_X, self.CAT_Y = (0, self.HEIGHT - 1) self.CHEESE_X, self.CHEESE_Y = np.random.randint(0, 9, 2, 'int') #making sure cheese is not at obstacles for obs in self.OBSTACLES: if self.CHEESE_X == obs[0] and self.CHEESE_Y == obs[1]: #then shift it up self.CHEESE_Y -= 1 self.MOVES['cat'] = 100 self.MOVES['cat'] = 100 return self.get_state() #-----------------------------------------------Render the enviroment-----------------------------------------------# def render(self, i_episode=-1): ''' rendering the environment using pygame display ''' #drawing our agents self.MOUSE.draw(self.MOUSE_X, self.MOUSE_Y) self.CAT.draw(self.CAT_X, self.CAT_Y) self.DISPLAY.blit(self.CHEESE_IMG, (self.CHEESE_X * self.BLOCK_WIDTH, self.CHEESE_Y * self.BLOCK_HEIGHT)) #drawing obstacles for pos in self.OBSTACLES: pygame.draw.rect(self.DISPLAY, BLUE, [ pos[0] * self.BLOCK_WIDTH, pos[1] * self.BLOCK_HEIGHT, self.BLOCK_WIDTH, self.BLOCK_HEIGHT ]) if i_episode >= 0: self.display_episode(i_episode) #--------------------------------------Agents takes step and the environment changes------------------------------------# def step(self, mouse_action, cat_action): reward = {'mouse': -1, 'cat': -1} done = False info = { 'cheese_eaten': False, 'mouse_caught': False, 'x': -1, 'y': -1, 'width': self.BLOCK_WIDTH, 'height': self.BLOCK_HEIGHT } #decreasing the no. of moves self.MOVES['cat'] -= 1 self.MOVES['mouse'] -= 1 #done if moves = 0 if self.MOVES['cat'] == 0 or self.MOVES['mouse'] == 0: done = True self.update_positions(mouse_action, cat_action) #mouse reached the cheese if self.MOUSE_X == self.CHEESE_X and self.MOUSE_Y == self.CHEESE_Y: done = True reward['mouse'] = 50 info['cheese_eaten'], info['x'], info[ 'y'] = True, self.MOUSE_X, self.MOUSE_Y #cat caught the mouse if self.CAT_X == self.MOUSE_X and self.CAT_Y == self.MOUSE_Y: done = True reward['cat'] = 50 reward['mouse'] = -20 info['mouse_caught'], info['x'], info[ 'y'] = True, self.MOUSE_X, self.MOUSE_Y for obs in self.OBSTACLES: if self.MOUSE_X == obs[0] and self.MOUSE_Y == obs[1]: reward['mouse'] = -20 self.MOUSE_X, self.MOUSE_Y = (0, 0) if self.CAT_X == obs[0] and self.CAT_Y == obs[1]: reward['cat'] = -20 self.CAT_X, self.CAT_Y = (0, self.HEIGHT - 1) return self.get_state(), reward, done, info #----------------------------Helper function to display episode-------------------------# def display_episode(self, epsiode): font = pygame.font.SysFont(None, 25) text = font.render("Episode: " + str(epsiode), True, TEXT_COLOR) self.DISPLAY.blit(text, (1, 1)) #-------------------------------Decide position changes based on action taken--------------# def get_changes(self, action): x_change, y_change = 0, 0 #decide action if action == 0: x_change = -1 #moving left elif action == 1: x_change = 1 #moving right elif action == 2: y_change = -1 #moving upwards elif action == 3: y_change = 1 #moving downwards return x_change, y_change #-----------------------------update positions of agents-------------------------------------# def update_positions(self, mouse_action, cat_action): x_change_mouse, y_change_mouse = self.get_changes(mouse_action) x_change_cat, y_change_cat = self.get_changes(cat_action) self.MOUSE_X += x_change_mouse self.MOUSE_Y += y_change_mouse self.CAT_X += x_change_cat self.CAT_Y += y_change_cat self.MOUSE_X, self.MOUSE_Y = self.fix(self.MOUSE_X, self.MOUSE_Y) self.CAT_X, self.CAT_Y = self.fix(self.CAT_X, self.CAT_Y) #------------------------------Push them back in to fight! There's no escape--------------------# def fix(self, x, y): # If agents out of bounds, fix! if x < 0: x = 0 elif x > self.WIDTH - 1: x = self.WIDTH - 1 if y < 0: y = 0 elif y > self.HEIGHT - 1: y = self.HEIGHT - 1 return x, y
class AutoSimulation: def __init__(self): self.autos = [] self.events = [] self.generation = 0 self.dt = 0 self.top_auto = None self.slow_car_removed = False self.learning_rate = LEARNING_RATE self.viewer = TargetViewer(self) self.mouse = Mouse(self) self.track = RaceTrack(self) self.text = Text("Generation : 0") self.start_time = pygame.time.get_ticks() self.addNewAutos(num_car) def generateTrack(self, difficulty): size = 700 # make it fixed size, previous code was : #randint(int(800 - (400 * difficulty)), 800) width = size // 4 # original is 4 self.track.generateTrack( Rect((size, size), (-size / 2.0, -size / 2.0)), width) def addNewAutos(self, n): for _ in range(n): self.autos.append(Auto(self)) def getFittestAuto(self): max_fit = -1 top_autos = [] for auto in self.autos: auto.top = 0 if auto.fitness > max_fit: max_fit = auto.fitness top_autos = [auto] if not auto.stop: pass if auto.fitness == max_fit: top_autos.append(auto) if len(top_autos) == 1: return top_autos[0] gate_pos = top_autos[0].getCurrentGate().next_track.gate.getCenter() closest = None top_auto = None for auto in top_autos: dis = gate_pos.getDis(auto.pos) if closest is None or dis < closest: closest = dis top_auto = auto top_auto.top = 1 self.top_auto = top_auto def start(self): difficulty = 1.0 - (1.0 / ((self.generation / 100.0) + 1.0)) self.generateTrack(difficulty) self.text = Text("Generation : " + str(self.generation)) self.start_time = pygame.time.get_ticks() for car in self.autos: car.start() def update(self): for car in self.autos: if not car.stop: # only update unstopped car car.update() if (not self.slow_car_removed) and ( (pygame.time.get_ticks() - self.start_time) > 10000): """remove slow car after N ms""" if car.vel.getMag() < 10: car.stop = 1 self.getFittestAuto() self.mouse.update() #if not self.top_auto.stop: # self.viewer.updatePos(self.top_auto.body.pos) # self.viewer.updatePos(None) # KERU fix the view at the center if pygame.time.get_ticks() - self.start_time > reset_timer: self.reset() def cont(self): """count stopped car and decide to continue the race""" stopped = 0 # was 0, but 1 is a dumb hack to stop the race if only 1 is running for car in self.autos: # count stopped car stopped += car.stop all_stopped = (stopped == (len(self.autos))) return 1 - all_stopped def end(self): new_batch = [] # update the learning rate if self.learning_rate > LEARNING_RATE_MIN: self.learning_rate -= LEARNING_RATE_DEC else: self.learning_rate = LEARNING_RATE_MIN print("Learning rate : %.4f" % self.learning_rate) #add the top auto to next run new_batch.append(self.top_auto) # count running car, for display running_car_count = 0 for running_auto in self.autos: if running_auto.stop == 0: running_car_count += 1 print("running cars : ", running_car_count) # count car that are still running and send them for the next run running_car_count = 0 for running_auto in self.autos: if running_auto.stop == 0 and running_car_count < num_car // 2: running_car_count += 1 ## make slightly less mutated child new_batch.append(running_auto.makeChild(self.learning_rate)) #complete the list up to 50% with mutated top car new_top_car = 0 while running_car_count < (num_car // 2): running_car_count += 1 new_top_car += 1 new_batch.append(self.top_auto.makeChild(self.learning_rate)) # complete the remaining with the list with mutated version of random car of this run # leave one for top car for i in range(len(self.autos) - running_car_count - 1): #auto = self.top_auto.makeChild() auto = choice(self.autos).makeChild(self.learning_rate) new_batch.append(auto) # finally, copy the new batch + the top auto (without mutating it) to the next batch self.autos = new_batch + [self.top_auto] self.generation += 1 print("Generation:", self.generation - 1, "Done!") def render(self): for car in self.autos: if not car.stop: # only draw car that are still running car.draw() self.top_auto.draw() self.top_auto.brain.draw( Rect((300, 200), (self.viewer.pos.x + 15, self.viewer.pos.y + 5))) self.track.draw() self.mouse.draw() self.text.draw(self.viewer.display) self.viewer.draw(self.top_auto.inputs) #enable to see sensor self.viewer.render() self.viewer.clear() def reset(self): self.end() self.start()
world = Map(Rect(0, 0, 600, 550)) world.load('mapdata.xml') world.sprites = spriteset world.start_fight = lambda h, m: draw_stack.open(win.CombatWindow(h, m)) gui.add(world) sidebar = StatsBar(Rect(620, 20, 150, 300), spriteset) world.tile_selected = sidebar.display gui.add(sidebar) game_over = False while not game_over: world.update() current_gui = draw_stack.top c = current_gui.get_cursor_at(mouse.get_pos()) # XXX draw all the objects here display.fill((0, 0, 0)) current_gui.draw(display) mouse.draw(display, c) # overlays = pygame.sprite.RenderUpdates() # overlays.draw(screen) pygame.display.flip() clock.tick(15) for event in pygame.event.get(): if event.type == pygame.locals.QUIT: game_over = True current_gui.process_event(event)
class Game: def __init__(self, total_episodes: int): self.window_width = constant.WIDTH * constant.TILE self.window_height = constant.HEIGHT * constant.TILE self._running = True self._display = None self._snake = None self._mouse = None self.episode = 1 self.total_episodes = total_episodes self.score = 0 self.max_score = 0 self.frames = 0 self.game_stats = [] self.specs = [] self.test_run = False self.snake = Snake() self.mouse = Mouse(constant.WIDTH, constant.HEIGHT, self.snake.body_position()) self.q = QLearning() def initialize_pygame(self): """ Initialize pygame along with display and image settings """ pygame.init() self._display = pygame.display.set_mode( (self.window_width, self.window_height), pygame.HWSURFACE) pygame.display.set_caption('SNAKE ' + 'Episode ' + str(self.episode)) self._snake = pygame.image.load("img/snake_body_mini.png").convert() # source for mouse: http://pixelartmaker.com/art/3d272b1bf180b60.png self._mouse = pygame.image.load("img/mouse_mini.png").convert() def game_over(self, collision_type: str): """ Print game results and exit the game """ collision_value = -1 # represents body collision if collision_type == 'the wall': collision_value = 1 self.snake.update_tail() self._running = False if self.score > self.max_score: self.max_score = self.score self.game_stats.append([self.frames, self.score, collision_value]) self.display(collision_type) self.next_episode() def display(self, collision_type: str): """ Displays game over status and scores, and can call display/save data functions :param collision_type: what type of collision ended the game """ if self.episode % constant.SAVE_EPISODE == 0: self.q.save_table(self.episode, clear_dir=constant.DELETE_JSON) print(f'GAME OVER! Snake collided with {collision_type}') print(f'SCORE: {self.score}') def move_snake(self, ai_play: bool): """ Check whether the snake has eaten the mouse or encountered a collision :param ai_play: True if ai play, False otherwise """ self.snake.update_head() # if snake eats mouse if self.snake.eats_mouse(self.mouse.x, self.mouse.y): self.mouse.generate_mouse(self.snake.body_position()) self.score += 1 if ai_play: self.q.update_reward('mouse') # if snake collides with itself elif self.snake.body_collision(): if ai_play: self.q.update_reward('snake') self.game_over('itself') # if snake collides with walls elif self.snake.wall_collision(0, self.window_width, 0, self.window_height): if ai_play: self.q.update_reward('wall') self.game_over('the wall') else: if ai_play: self.q.update_reward('empty') self.snake.update_tail() def abs_coordinates(self): snake_head = self.snake.head_coordinates() mouse_loc = self.mouse.relative_coordinates(snake_head) tail_loc = self.snake.tail_coordinates() return tail_loc, mouse_loc def render(self): """ Render the visual components of the game """ self._display.fill((0, 0, 0)) self.snake.draw(self._display, self._snake) self.mouse.draw(self._display, self._mouse) pygame.display.flip() def human_play(self, delay: int): """ Executes the game play, snake movements, and loops until the game ends. Keys can be used to play the game. :param delay: defines the frame delay with lower values (e.g. 1) resulting in a fast frame, while higher values (e.g. 1000) result in very slow frames """ while self._running: pygame.event.pump() keys = pygame.key.get_pressed() if keys[pygame.K_RIGHT]: self.snake.set_east() elif keys[pygame.K_LEFT]: self.snake.set_west() elif keys[pygame.K_UP]: self.snake.set_north() elif keys[pygame.K_DOWN]: self.snake.set_south() elif keys[pygame.K_ESCAPE]: self._running = False self.move_snake(False) self.render() sleep(float(delay) / 1000) self.frames += 1 def set_direction(self, direction: str): """ Sets the direction for the snake to take :param direction: specified direction """ if direction == 'east': self.snake.set_east() elif direction == 'west': self.snake.set_west() elif direction == 'north': self.snake.set_north() else: # south self.snake.set_south() def ai_train(self, delay: int, resume_state: bool): """ Executes the AI training, looping until the snake is trained the total number of episodes. Movements are implemented by the AI rather than by a human pressing keys. :param delay: defines the frame delay with lower values (e.g. 1) resulting in a fast frame, while higher values (e.g. 1000) result in very slow frames :param resume_state: if True, start training from externally saved table's next episode, if False, initial episode is 1 """ # If resuming from a saved state, start from the loaded state's next episode if resume_state: self.resume_game(self.total_episodes) while self._running: pygame.event.pump() tail_loc, mouse_loc = self.abs_coordinates() snake_direction = self.snake.current_direction() state = self.q.define_state(tail_loc, mouse_loc, snake_direction) action = self.q.select_action(state) self.set_direction(action) self.move_snake(True) tail_loc, mouse_loc = self.abs_coordinates() snake_direction = self.snake.current_direction() next_state = self.q.define_state(tail_loc, mouse_loc, snake_direction) self.q.update(state, next_state, action) self.q.reset_reward() self.render() sleep(float(delay) / 1000) self.frames += 1 def ai_test(self, delay: int, resume_state: bool): """ Tests the AI on previous training data :param delay: defines the frame delay :param resume_state: if True, start training from externally saved table's next episode, if False, initial episode is 1 """ self.test_run = True self.episode = 1 # If resuming from a saved state, start from the loaded state's next episode if resume_state: self.resume_game(constant.TOTAL_TESTS) if constant.PARAM_TEST: self.total_episodes = constant.TOTAL_TESTS # Run the total number of tests specified while self.episode <= self.total_episodes: caption = 'SNAKE ' + 'FINAL TEST RUN: EPISODE ' + str(self.episode) self.reset_game(caption) self.game_stats = [] self.specs = [] while self._running: pygame.event.pump() tail_loc, mouse_loc = self.abs_coordinates() snake_direction = self.snake.current_direction() state = self.q.define_state(tail_loc, mouse_loc, snake_direction) action = self.q.select_action(state) self.set_direction(action) self.move_snake(True) self.render() sleep(float(delay) / 1000) self.frames += 1 print( f'(TEST RUN EPISODE {str(self.episode)}) FINAL SCORE: {self.score}, FINAL MAX SCORE: {self.max_score}\n' ) self.episode += 1 def resume_game(self, total_tests): filename = 'episode' + str(constant.RESUME_EPISODE) + '.json' self.episode = self.q.load_table(filename) if self.episode < 1: print(f'Table failed to load') self.total_episodes = self.episode + total_tests - 1 def reset_game(self, caption: str): pygame.display.set_caption(caption) self.score = 0 self.frames = 0 self._running = True self.snake.initialize_positions(self.mouse.x, self.mouse.y) self.mouse.generate_mouse(self.snake.body_position()) def next_episode(self): """ Sets-up the next episode or completes the final episode """ if self.episode >= self.total_episodes: self.prep_data() return # set new episode self.episode += 1 print(f'\nNEW GAME, EPISODE {self.episode}') caption = 'SNAKE ' + 'Episode ' + str(self.episode) self.reset_game(caption) def prep_data(self): """ Prepares data formatting with headers, specific test names, etc """ self.specs = [] filename = '' if self.test_run: filename = 'testing_' + constant.PARAM + str(constant.PARAM_VAL) if constant.PARAM_TEST: filename += constant.PARAM + str(constant.PARAM_VAL) stats_file = filename + '_data.csv' header = ['Steps', 'Scores', 'Collisions'] self.write_data(stats_file, header, self.game_stats) specs_file = filename + '_specs.csv' header = ['Parameters', 'Values'] self.specs.append(['total episodes', self.episode]) self.specs.append(['height', constant.HEIGHT]) self.specs.append(['width', constant.WIDTH]) self.specs.append(['learning rate', constant.ETA]) self.specs.append(['discount', constant.DISCOUNT]) self.specs.append(['epsilon', constant.EPSILON]) self.specs.append(['mouse reward', constant.MOUSE]) self.specs.append(['wall penalty', constant.WALL]) self.specs.append(['self-collision penalty', constant.SNAKE]) self.specs.append(['empty tile penalty', constant.EMPTY]) self.write_data(specs_file, header, self.specs, True) def write_data(self, filename: str, header: [str], data: [], add_specs: bool = False): """ Writes the data from the current session to a file. :param filename: filename to write data :param header: header names for data :param data: data to add to file :param add_specs: True if writing specs file, False otherwise """ op = 'w' # default write to CSV path = constant.DATA_DIR file = path + filename # create directory if it doesn't exist if not os.path.exists(path): os.mkdir(path) # append data to existing file if constant.RESUME and os.path.isfile(file): op = 'a' # write specs if add_specs: op = 'w' # write data to csv file(s) with open(file, op, newline='') as outfile: w = csv.writer(outfile) if not constant.RESUME: w.writerow(header) if not constant.PARAM_TEST and constant.RESUME: w.writerow(header) w.writerows(data) outfile.close()