def update_Q(sn, di, f, ss, alpha):
old_state = get_state(sn, sn[1], di[1], f, ss)
new_state = get_state(sn, sn[0], di[0], f, ss)
if di[1] == di[0]:
old_act = 0
elif (di[1] == up and di[0] == left) or (di[1] == left and di[0] == down) \
or (di[1] == down and di[0] == right) or (di[1] == right and di[0] == up):
old_act = 1
else:
old_act = 2
# print(space.shape, old_state)
space[old_state] = 1
max_Q = np.max(Q[new_state])
if adjacent(sn[0], f, ss, ss, dir_=di[0]):
Q[old_state][old_act] = (1 - alpha) * Q[old_state][old_act] + alpha * (
FOOD_R + GAMMA * max_Q)
return FOOD_R
elif collision(sn, di[0], ss):
Q[old_state][old_act] = (1 - alpha) * Q[old_state][old_act] + alpha * (
DEATH_R + GAMMA * max_Q)
return DEATH_R
else:
Q[old_state][old_act] = (1 - alpha) * Q[old_state][old_act] + alpha * (
LIVING_R + GAMMA * max_Q)
return LIVING_R
def update_Q_sequence(sn, di, f, ss, alpha, depth, past_rewards):
old_state = get_state(sn, sn[depth - 1], di[depth - 1], f, ss)
new_state = get_state(sn, sn[0], di[0], f, ss)
if di[depth - 1] == di[depth - 2]:
old_act = 0
elif (di[depth-1] == up and di[depth-2] == left) or (di[depth-1] == left and di[depth-2] == down) \
or (di[depth-1] == down and di[depth-2] == right) or (di[depth-1] == right and di[depth-2] == up):
old_act = 1
else:
old_act = 2
space[old_state][old_act] = 1
reduced_Q = reduce_Q(np.max(Q[new_state]), past_rewards)
if adjacent(sn[0], f, ss, ss, dir_=di[0]):
Q[old_state][old_act] = (1 - alpha) * Q[old_state][old_act] + alpha * (
FOOD_R + GAMMA * reduced_Q)
return FOOD_R
elif collision(sn, di[0], ss):
Q[old_state][old_act] = (1 - alpha) * Q[old_state][old_act] + alpha * (
DEATH_R + GAMMA * reduced_Q)
return DEATH_R
else:
Q[old_state][old_act] = (1 - alpha) * Q[old_state][old_act] + alpha * (
LIVING_R + GAMMA * reduced_Q)
return LIVING_R
def add_exp(sn, di, f, ss):
old_state = get_state(sn, sn[1], di[1], f, ss)
new_state = get_state(sn, sn[0], di[0], f, ss)
if di[1] == di[0]:
old_act = 0
elif (di[1] == up and di[0] == left) or (di[1] == left and di[0] == down) \
or (di[1] == down and di[0] == right) or (di[1] == right and di[0] == up):
old_act = 1
else:
old_act = 2
if adjacent(sn[0], f, ss, ss, dir_=di[0]):
r = FOOD_R
elif collision(sn, di[0], ss):
r = DEATH_R
else:
r = LIVING_R
global experience
if experience is None:
experience = np.array([[old_state, old_act, r, new_state]])
else:
experience = np.append(experience,
[[old_state, old_act, r, new_state]],
axis=0)
if len(experience) > MEM_LENGTH:
np.delete(experience, 0, 0)
def get_acts(snake, snake_dirs, ss, f, depth):
act = None
min_ = math.inf
if snake_dirs[0] == left or snake_dirs[0] == right:
dirs = [snake_dirs[0], up, down]
else:
dirs = [snake_dirs[0], right, left]
for a in dirs:
n_snake, n_dirs = update_snake(deepcopy(snake), deepcopy(snake_dirs),
a, ss)
coll = collision(n_snake, n_dirs[0], ss)
if not coll:
val = math.sqrt((n_snake[0].x - f.x)**2 + (n_snake[0].y - f.y)**2)
else:
val = math.inf
if depth != 1 and val != 0 and not coll:
val += get_acts(n_snake, n_dirs, ss, f, depth - 1)[1]
if val < min_:
act = a
min_ = val
return act, min_
def on_draw(self):
self.clear()
draw_axis(3, (-1, 1, -1))
self.set3d()
self.camera.move()
self.light.show()
self.ground.draw()
for i in range(len(self.models)):
for j in range(len(self.models)):
if i != j:
collision(self.models[i], self.models[j])
ground_collision(self.models[i], self.ground)
for model in self.models:
model.draw()
def _collision(self, coords):
"""Piece._collision(coords) -> bool.
Returns True iff coords are out of board's boundaries or
if coords would result in collision with pile.
"""
return utils.collision(coords,
self.board().pile().coordinates(),
self.board().size())
def check_collisions(self):
# Double loop yay
for player in self.players:
for obstacle in self.obstacles:
player_coords = self.canvas.coords(player.canvas_object)
# We need to get the coordinates of top left and bottom right corners of the obstacle
obstacle_coords = (obstacle['x'], obstacle['y'],
obstacle['x'] + obstacle['w'],
obstacle['y'] + obstacle['h'])
if utils.collision(player_coords, obstacle_coords):
player.kill()
self.canvas.itemconfig(player.canvas_object, fill='black')
def detect_collision(self, objects, level, msglog):
center_x1 = self.xpos + self.img.get_rect()[2] / 2
center_y1 = self.ypos + self.img.get_rect()[3] / 2
for i in range(len(objects)):
center_x2 = objects[i].xpos + objects[i].img.get_rect()[2] / 2
center_y2 = objects[i].ypos + objects[i].img.get_rect()[3] / 2
dist = (self.img.get_rect()[2] +
objects[i].img.get_rect()[2]) * 0.93 / 2
if collision(center_x1, center_y1, center_x2, center_y2, dist):
if self.durability > 1:
self.durability -= 1
else:
level.state = -1
objects[i].explode()
pygame.mixer.Sound("musics/ship_damaged.wav").play()
msglog.update("SHIP HAS BEEN HIT", (255, 0, 0))
def get_state(sn, seg, d, f, ss):
# ret = []
ret = [0, 0, 0]
sides = get_rel_in_glo(d)
blocked = collision(sn, d, ss, blocking=True)
for side in blocked:
if side in sides:
ret[sides.index(side)] = 1
# com = (sum([i.x for i in sn]) // len(sn), sum([i.y for i in sn]) // len(sn))
for l in [(f.x, f.y)]:
if (d == up and l[1] == seg.y and l[0] > seg.x) or (d == left and l[0] == seg.x and l[1] < seg.y) \
or (d == down and l[1] == seg.y and l[0] < seg.x) or (d == right and l[0] == seg.x and l[1] > seg.y):
ret.append(0)
elif (d == up and l[0] == seg.x and l[1] < seg.y) or (d == down and l[0] == seg.x and l[1] > seg.y) \
or (d == left and l[1] == seg.y and l[0] < seg.x) or (d == right and l[1] == seg.y and l[0] > seg.x):
# ret.append(2)
ret.append(1)
elif (d == up and l[1] == seg.y and l[0] < seg.x) or (d == left and l[0] == seg.x and l[1] > seg.y) \
or (d == down and l[1] == seg.y and l[0] > seg.x) or (d == right and l[0] == seg.x and l[1] < seg.y):
# ret.append(4)
ret.append(2)
elif (d == up and l[0] == seg.x and l[1] > seg.y) or (d == left and l[1] == seg.y and l[0] > seg.x) \
or (d == down and l[0] == seg.x and l[1] < seg.y) or (d == right and l[1] == seg.y and l[0] < seg.x):
# ret.append(6)
ret.append(3)
elif (d == up and l[0] > seg.x and l[1] < seg.y) or (d == left and l[0] < seg.x and l[1] < seg.y) \
or (d == down and l[0] < seg.x and l[1] > seg.y) or (d == right and l[0] > seg.x and l[1] > seg.y):
# ret.append(1)
ret.append(0)
elif (d == up and l[0] < seg.x and l[1] < seg.y) or (d == left and l[0] < seg.x and l[1] > seg.y) \
or (d == down and l[0] > seg.x and l[1] > seg.y) or (d == right and l[0] > seg.x and l[1] < seg.y):
# ret.append(3)
ret.append(1)
elif (d == up and l[0] < seg.x and l[1] > seg.y) or (d == left and l[0] > seg.x and l[1] > seg.y) \
or (d == down and l[0] > seg.x and l[1] < seg.y) or (d == right and l[0] < seg.x and l[1] < seg.y):
# ret.append(5)
ret.append(2)
else:
# ret.append(7)
ret.append(3)
print(tuple(ret))
return tuple(ret)
def detect_collision(self, objects, player, energy_gauge):
for i in range(self.ammo):
if self.onfire:
center_x1 = self.xpos + self.img.get_rect()[2] / 2
center_y1 = self.ypos + self.img.get_rect()[3] / 2
for j in range(len(objects)):
center_x2 = objects[j].xpos + objects[j].img.get_rect()[2] / 2
center_y2 = objects[j].ypos + objects[j].img.get_rect()[3] / 2
dist = (self.img.get_rect()[2] + objects[j].img.get_rect()[2]) * 0.93 / 2
if collision(center_x1, center_y1, center_x2, center_y2, dist):
self.onfire = False
objects[j].explosion.frame = 0
if objects[j].hp <= self.damage:
objects[j].explosion.size = "large"
objects[j].explode()
pygame.mixer.Sound("musics/asteroid_collision.wav").play(0, 0, 400)
energy_gauge.update(player.energy*2.5)
else:
objects[j].hp -= self.damage
objects[j].explosion.size = "small"
objects[j].explosion.show(self.xpos + 16, self.ypos + 16)
pygame.mixer.Sound("musics/drone_collision.wav").play(0, 0, 400)
break
def detect_collision(self, objects):
for i in range(self.count):
if self.onfire[i]:
center_x1 = self.xpos[i] + self.img.get_rect()[2] / 2
center_y1 = self.ypos[i] + self.img.get_rect()[3] / 2
for j in range(len(objects)):
center_x2 = objects[j].xpos + objects[j].img.get_rect(
)[2] / 2
center_y2 = objects[j].ypos + objects[j].img.get_rect(
)[3] / 2
dist = (self.img.get_rect()[2] +
objects[j].img.get_rect()[2]) * 0.93 / 2
if collision(center_x1, center_y1, center_x2, center_y2,
dist):
if self.name == settings.WEAPON_LIST[0]:
self.onfire[i] = False
if self.name == settings.WEAPON_LIST[
1] and self.damage < objects[j].hp:
self.onfire[i] = False
objects[j].explosion.frame = 0
if objects[j].hp <= self.damage:
objects[j].explosion.size = "large"
objects[j].explode()
pygame.mixer.Sound(
"musics/asteroid_collision.wav").play(
0, 0, 400)
else:
objects[j].hp -= self.damage
objects[j].explosion.size = "small"
objects[j].explosion.show(self.xpos[i] + 16,
self.ypos[i] + 16)
pygame.mixer.Sound(
"musics/drone_collision.wav").play(0, 0, 400)
if self.name == settings.WEAPON_LIST[2]:
self.detonate = True
break
score += 1
# Update directions
t1 = new_dir
for i in range(len(snake_dirs)):
t2 = snake_dirs[i]
snake_dirs[i] = t1
t1 = t2
# Update positions
for i in range(len(snake)):
snake[i].move_ip(step_size * snake_dirs[i][0], step_size * snake_dirs[i][1])
# print(snake_dirs)
if CONTROL == "q_ai" and collision(snake, snake_dirs[0], step_size):
if TESTING:
eps, alpha = 0, 0
else:
eps, alpha = q_ai.get_eps(epis), q_ai.get_alpha(epis)
# cur_rew = q_ai.update_Q(snake, snake_dirs, food_rect, step_size, alpha)
cur_rew = q_ai.update_Q_sequence(snake, snake_dirs, food_rect, step_size, alpha, start_size, reward_buffer)
epi_rew += cur_rew
reward_buffer.insert(0, cur_rew)
if len(reward_buffer) > start_size:
reward_buffer.pop()
t, new_dir, snake, snake_dirs, score, f_pos = init_game(start_size, step_size, block_rect, True)
food_rect.x, food_rect.y = f_pos[0], f_pos[1]
print("Episode {} terminated after {} cycles".format(epis, c))
print("Episode {} reward: {}".format(epis, epi_rew))