class App:
windowDimY = 14
windowDimX = 18
windowWidth = 0
windowHeight = 0
toolbarWidth = 200
player = 0
apple = 0
toolbar = 0
dataCollect = 0
displayq = False
def __init__(self,dq,state,freq):
self._running = True
self._display_surf = None
self._image_surf = None
self._apple_surf = None
self.game = Game()
self.player = Player(3,self.windowDimX, self.windowDimY)
self.windowWidth = self.windowDimX*self.player.step
self.windowHeight = self.windowDimY*self.player.step
self.toolbar = Toolbar(self.toolbarWidth, self.windowWidth, self.windowHeight)
self.apple = Apple(randint(0,self.windowDimX-1), randint(0,self.windowDimY-1))
self.dataCollect = DataCollector()
self.displayq=dq
self.state_size = state
self.frequency = freq
def on_init(self):
pygame.init()
if(self.displayq):
self._display_surf = pygame.display.set_mode((self.windowWidth+self.toolbarWidth,self.windowHeight), pygame.HWSURFACE)
pygame.display.set_caption('Pygame Snake game!')
self._image_surf = pygame.image.load("images/game_objects/smake.png").convert()
self._apple_surf = pygame.image.load("images/game_objects/smapple.png").convert()
self.toolbar.load_images()
self._running = True
# self.savepath = "frames/"+time.strftime("%Y%m%d-%H%M%S")
# os.mkdir(self.savepath)
def on_event(self, event):
if event.type == QUIT:
self._running = False
def on_loop(self):
self.player.update()
# does snake collide with itself?
for i in range(2,self.player.length):
if self.game.isCollision(self.player.x[0],self.player.y[0],self.player.x[i], self.player.y[i],self.player.step-1):
print("You lose! Collision with yourself: ")
print("x[0] (" + str(self.player.x[0]) + "," + str(self.player.y[0]) + ")")
print("x[" + str(i) + "] (" + str(self.player.x[i]) + "," + str(self.player.y[i]) + ")")
self.player.crashed = True
# does snake eat apple?
for i in range(0,self.player.length):
if self.game.isCollision(self.apple.x,self.apple.y,self.player.x[i], self.player.y[i],self.player.step-1):
self.apple.x = randint(0,self.windowDimX-1) * self.player.step
self.apple.y = randint(0,self.windowDimY-1) * self.player.step
print("apple x=",self.apple.x,"apple y=",self.apple.y)
self.player.eatenApple = True
#does snake collide with wall?
if(self.player.x[0]<0 or self.player.x[0]>=self.windowWidth or self.player.y[0]<0 or self.player.y[0]>=self.windowHeight):
print("You lose! Collision with wall: ")
print("x[0] (" + str(self.player.x[0]) + "," + str(self.player.y[0]) + ")")
self.player.crashed = True
pass
def on_render(self, state):
self._display_surf.fill((0,0,0))
self.player.draw(self._display_surf, self._image_surf)
self.apple.draw(self._display_surf, self._apple_surf)
self.toolbar.draw(self._display_surf, self.player.direction, state)
pygame.display.flip()
def on_cleanup(self):
pygame.quit()
# initializing agent before the 1st move
def init_agent(self, agent):
state_init1 = agent.get_state(game, self.player, self.food) #first state after random placement
#first action
def reset_player(self):
self.player = Player(3,self.windowDimX, self.windowDimY)
#print(self.player.x)
#print(self.player.y)
def on_execute(self,speed):
print('starting execution!')
params = parameters()
agent = dqnagent(params,self.state_size) #initialize the agent!
if(agent.load_weights): #load weights maybe
agent.model.load_weights(agent.weights)
print("loaded the weights")
counter = 0 #how many games have been played/trials done
record = 0 #highest score
#---------------------------------------------------------------------------
#------------------------- LOOP THRU EPISODES ------------------------------
#---------------------------------------------------------------------------
while counter<params['episodes']: #still have more trials to do
counter += 1
print(counter)
print("\nEPISODE ", counter, "\n")
if not params['train']: # if you're not training it, no need for exploration
agent.epsilon = 0
else:
agent.epsilon = 1.0 - ((counter -1) * params['epsilon_decay_linear'])#exploration/randomness factor that decreases over time
#print("EPSILON = ", agent.epsilon, "\n")
if self.on_init() == False:
self._running = False
#print("PLAYER\tx : ", self.player.x,"\ty : ", self.player.y, "\n")
duration = 0
#---------------------------------------------------------------------------
#--------------------------- INDIVIDUAL EPISODE ----------------------------
#---------------------------------------------------------------------------
# indexx = 0
while(self._running):
if(counter%self.frequency==0):
self.dataCollect.record(self.player.x, self.player.y, self.apple.x, self.apple.y)
duration+=1
#print("\nMOVE : ", duration, "\n")
if(self.displayq):
pygame.event.pump()
keys = pygame.key.get_pressed()
if (keys[K_ESCAPE]):
exit(0)
oldstate = agent.get_state(self, self.player, self.apple)
#print("\noldstate = ", oldstate)
#--------------------------- GET AGENT ACTION ----------------------------
if random() < agent.epsilon: #every so often random exploration
action = randint(0,3) #random action
#print("random action : ",action)
else: #Actionprecited by agent
state = oldstate.reshape(1,self.state_size**2+8)
predictedq= agent.model.predict(state) # predicts the q values for the action in that state
action = np.argmax(predictedq[0]) #maximum (highest q) action
#print("predicted action : ", action, "\tq-values : ", predictedq)
#---------------------------- EXECUTE ACTION -----------------------------
print(action)
self.player.do_move(action) #do the action
self.on_loop()
newstate = agent.get_state(self, self.player, self.apple) #new state from the action we've taken
reward = agent.set_reward(self.player)
#print("newstate = ", newstate)
#print("reward = ", reward)
#print("crashed = ", self.player.crashed, "\n")
#---------------------------- SHORT TRAINING -----------------------------
if(params['train']):
agent.train_short_memory(oldstate,action, reward, newstate, self.player.crashed)
agent.remember(oldstate,action, reward, newstate, self.player.crashed)
#------------------------------ RENDER GAME ------------------------------
self._running = not(self.player.crashed)
if(self.displayq):
self.on_render(newstate)
# if(counter%self.frequency==0):
# self.on_render(newstate)
# pygame.image.save(self._display_surf,savepath+str(indexx))
time.sleep (speed/1000.0)
# indexx +=1
#---------------------------------------------------------------------------
#----------------------- TRAINING & DATA COLLECTION ------------------------
#---------------------------------------------------------------------------
if(params['train']):
agent.replay_new(agent.memory, params['batch_size'])
# self.dataCollect.add(self.player.length,duration,agent.epsilon,agent.history.losses)
self.dataCollect.add(self.player.length,duration,agent.epsilon, 0.0)
self.dataCollect.save()
#print(agent.history.losses.length())
#agent.history.losses = []
self.player.reset(3,self.windowDimX, self.windowDimY )
self.on_cleanup()
#---------------------------------------------------------------------------
#------------------------------ DATA OUTPUT --------------------------------
#---------------------------------------------------------------------------
if(params['train']):
os.mkdir(params['weights_path_save'])
agent.model.save_weights(params['weights_path_save']+'/weights.hdf5')
self.dataCollect.save()
class App:
windowDimY = 14
windowDimX = 18
windowWidth = 0
windowHeight = 0
toolbarWidth = 200
player = 0
apple = 0
toolbar = 0
def __init__(self):
self._running = True
self._display_surf = None
self._image_surf = None
self._apple_surf = None
self.game = Game()
self.player = Player(3, self.windowDimX, self.windowDimY)
self.windowWidth = self.windowDimX * self.player.step
self.windowHeight = self.windowDimY * self.player.step
self.toolbar = Toolbar(self.toolbarWidth, self.windowWidth,
self.windowHeight)
self.apple = Apple(randint(0, self.windowDimX - 1),
randint(0, self.windowDimY - 1))
def on_init(self):
pygame.init()
self._display_surf = pygame.display.set_mode(
(self.windowWidth + self.toolbarWidth, self.windowHeight),
pygame.HWSURFACE)
pygame.display.set_caption('Pygame Snake game!')
self._running = True
self._image_surf = pygame.image.load(
"images/game_objects/smake.png").convert()
self._apple_surf = pygame.image.load(
"images/game_objects/smapple.png").convert()
self.toolbar.load_images()
def on_event(self, event):
if event.type == QUIT:
self._running = False
def on_loop(self):
self.player.update()
# does snake collide with itself?
for i in range(2, self.player.length):
if self.game.isCollision(self.player.x[0], self.player.y[0],
self.player.x[i], self.player.y[i],
self.player.step - 1):
print("You lose! Collision with yourself: ")
print("x[0] (" + str(self.player.x[0]) + "," +
str(self.player.y[0]) + ")")
print("x[" + str(i) + "] (" + str(self.player.x[i]) + "," +
str(self.player.y[i]) + ")")
exit(0)
# does snake eat apple?
for i in range(0, self.player.length):
if self.game.isCollision(self.apple.x, self.apple.y,
self.player.x[i], self.player.y[i],
self.player.step - 1):
self.apple.x = randint(0,
self.windowDimX - 1) * self.player.step
self.apple.y = randint(0,
self.windowDimY - 1) * self.player.step
print("apple x=", self.apple.x, "apple y=", self.apple.y)
self.player.eatenApple = True
#does snake collide with wall?
if (self.player.x[0] < 0 or self.player.x[0] >= self.windowWidth
or self.player.y[0] < 0
or self.player.y[0] >= self.windowHeight):
print("You lose! Collision with wall: ")
print("x[0] (" + str(self.player.x[0]) + "," +
str(self.player.y[0]) + ")")
exit(0)
pass
def state(self):
middle = int(dim / 2)
grid = np.zeros((dim, dim), dtype=bool)
print("x=", self.player.x, "y=", self.player.y)
relativex = [
int((a - self.player.x[0]) / self.player.step + middle)
for a in self.player.x
]
relativey = [
int((a - self.player.y[0]) / self.player.step + middle)
for a in self.player.y
]
for x, y in zip(relativex, relativey):
if (x >= 0 and y >= 0 and x < dim and y < dim):
grid[y, x] = True
print("x", relativex, "y", relativey)
right = int(self.windowDimX - (self.player.x[0] / self.player.step) +
middle)
down = int(self.windowDimY - (self.player.y[0] / self.player.step) +
middle)
up = int(middle - (self.player.y[0] / self.player.step))
left = int(middle - (self.player.x[0] / self.player.step))
if (right < dim and right >= 0):
grid[:, right:] = True
if (left < dim and left >= 0):
grid[:, :left] = True
if (down < dim and down >= 0):
grid[down:, :] = True
if (up < dim and up >= 0):
grid[:up, :] = True
print(right, left, up, down)
print("GRID after")
print(grid)
grid = list(grid.flatten())
state = [
self.player.direction == 0, #right
self.player.direction == 1, #left
self.player.direction == 2, #up
self.player.direction == 3, #down
self.apple.y < self.player.y[0], #food is up from the player
self.apple.y > self.player.y[0], #food is down from the player
self.apple.x <
self.player.x[0], #food is to the left of the player
self.apple.x >
self.player.x[0] #food is to the right of the player
]
state += grid
for i in range(len(state)): #convert list to 0s and 1s
if (state[i]):
state[i] = 1
else:
state[i] = 0
return np.asarray(state)
def on_render(self):
self._display_surf.fill((0, 0, 0))
self.player.draw(self._display_surf, self._image_surf)
self.apple.draw(self._display_surf, self._apple_surf)
print(self.state())
self.toolbar.draw(self._display_surf, self.player.direction,
self.state())
pygame.display.flip()
def on_cleanup(self):
pygame.quit()
def on_execute(self):
if self.on_init() == False:
self._running = False
while (self._running):
pygame.event.pump()
keys = pygame.key.get_pressed()
if (keys[K_RIGHT]):
self.player.moveRight()
if (keys[K_LEFT]):
self.player.moveLeft()
if (keys[K_UP]):
self.player.moveUp()
if (keys[K_DOWN]):
self.player.moveDown()
if (keys[K_ESCAPE]):
self._running = False
self.on_loop()
self.on_render()
time.sleep(70.0 / 1000.0)
self.on_cleanup()
