def draw(img):
# to ger real color we do this:
dino_coords = []
obstacle_coords = []
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
dino_coords = find.find_dino(img)
for i in range(1, types_of_obstacles+1):
obstacle_coords.append(find.find_obstacle(img, i))
screen = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
cv2.rectangle(screen, dino_coords[0][0], dino_coords[0][1], (0, 255, 0), 2)
for coords in obstacle_coords:
cv2.rectangle(screen, coords[0], coords[1], (0, 0, 255), 2)
# subprocess.call("clear")
# print(f'dino_coords: {dino_coords}, obstacle_coords: {obstacle_coords}')
return screen
def eval_genomes(genomes, config):
global frame, ROI, play_toggle, game, score_ROI, generation, max_fitness, winner, gamelog
print('playing')
last_dist = 0
tess_config = ('-l eng --oem 1 --psm 7')
score_tess_config = ('digits --oem 2 --psm 5')
gameover_ROI = [(50, 120), (300, 900)]
score_time = time.time()
force_gameover = False
generation += 1
# fgo_thresh = 350 + generation * 50
# fgo_thresh = 1800 if fgo_thresh > 1800 else fgo_thresh
reload()
print('reloaded page')
time.sleep(1)
for genome_id, genome in genomes:
genome.fitness = 0
net = neat.nn.FeedForwardNetwork.create(genome, config)
scroll_go = False
force_gameover = False
last_frame = False
first = True
score_time = time.time()
scores = []
# start game
pyautogui.press('up')
time.sleep(1)
pyautogui.press('up')
while True:
# Get raw pixels from the screen, save it to a Numpy array
try:
# screen shot
img = np.array(sct.grab(monitor))
screen = img.copy()
# roi of just obstacles
obstacles = img[ROI[0][0]:ROI[0][1], ROI[1][0]:ROI[1][1]]
# obstacles = img[43:256, 950:1300]
gray = cv2.cvtColor(obstacles, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(
gray, 90, 255,
cv2.THRESH_BINARY) # get binary image of abstacles
edged = cv2.Canny(
thresh, 30,
200) # edge detection to find obstacle hitboxes
# find contours of obstacles
contours, hierarchy = cv2.findContours(edged,
cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
# get hitboxes
points = findBoundingBoxesWithShift(contours)
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
dino_coords = find.find_dino(img)
# find dist between dino and nearest obstacle
dist, start_point, end_point, width = findDistance(
dino_coords, points)
# feed data to nn and run output
dino_y = start_point[-1]
cac_y = end_point[-1]
delta_dist = abs(last_dist - dist)
output = net.activate([dist, cac_y, dino_y, width, delta_dist])
# [0] jump, [1] crouch, [2] idle
if output[0] >= output[1] and output[0] >= output[
2]: # if jump is the highest value
pyautogui.press('up')
elif output[1] >= output[0] and output[1] >= output[2]:
pyautogui.press('down')
genome.fitness += 0.25
if cac_y > 310:
genome.fitness += 150
elif output[2] >= output[0] and output[2] >= output[1]:
pass
# check if game ended
if (len(points) >= 9 and len(points) < 20
and last_dist == dist) or force_gameover:
# Run tesseract OCR on image to see if words "game over" are on screen
gameover = img[gameover_ROI[0][0]:gameover_ROI[0][1],
gameover_ROI[1][0]:gameover_ROI[1][1]]
text = pytesseract.image_to_string(gameover,
config=tess_config)
# print('maybe gameover')
if (text.lower().replace(' ', '')
== "gameover") or force_gameover:
# for when nn presses down and scrolls under game over text
if scroll_go:
pyautogui.scroll(20, x=690, y=450)
time.sleep(1)
pyautogui.scroll(20, x=690, y=450)
print('scrolling')
time.sleep(0.3)
# total time the dino has ran for
time_score = time.time() - score_time
# make sure score isnt ''
score = ''
count = 0
while score == '' and count < 5:
pyautogui.scroll(10, x=690, y=450)
img = np.array(sct.grab(monitor))
score = getScore(img)
time.sleep(0.7)
count += 1
if count >= 4:
score = 0
# print(f'Bonus: {genome.fitness}')
try:
print(
f'Game Over! Game: {game} - \nScore: {int(score)}\nBonus: {genome.fitness}\nTime: {time_score}\nFitness: {genome.fitness + int(score)}',
end='\n\n')
gamelog.write(
f'Game Over! Game: {game} - ID: {genome_id}\nScore: {int(score)}\nBonus: {genome.fitness}\nTime: {time_score}\nFitness: {genome.fitness + int(score)}\n\n'
)
except:
print(f"{score} is not an integer")
# make sure game lasted over three seconds
if time_score >= 3:
game += 1
if not force_gameover or scroll_go:
# set fitness
genome.fitness += int(score)
if genome.fitness > max_fitness:
winner = genome
break
else:
# replay game
reload()
pyautogui.scroll(20, x=690, y=450)
pyautogui.press('up')
time.sleep(1)
pyautogui.press('up')
score_time = time.time()
force_gameover = False
if len(points) > 25: # if nn spams down and scrolls the page
force_gameover = True
scroll_go = True
print('scroll gameover')
time.sleep(0.5)
if int(time.time()) % 1000 == 0:
pyautogui.scroll(-10, x=690, y=450)
pyautogui.scroll(30, x=690, y=450)
print('auto scrolling')
if int(time.time()) % 750 < 3:
if first:
print('score check')
first = False
if last_frame:
score = getScore(screen)
if score != '':
scores.append(score)
last_frame = False
else:
last_frame = True
elif len(scores) != 0:
# print(scores)
cont = False
for i in range(len(scores)):
if scores[i - 1] == scores[i]:
continue
else:
cont = True
break
if not cont:
print('score check: failed')
force_gameover = True
else:
print('score check: passed')
scores = []
first = True
last_dist = dist
except KeyboardInterrupt:
print('program terminated: keyboard interrupt')
cv2.destroyAllWindows()
break
def calibrate():
global frame, ROI, play_toggle, game, score_ROI
last_dist = 0
tess_config = ('-l eng --oem 1 --psm 3')
gameover_ROI = [(50, 120), (300, 900)]
score_time = time.time()
force_gameover = False
print('Focus the chrome browser to start calibration')
time.sleep(5)
print('calibrating')
print('pressing up')
pyautogui.press('up')
print('pressed up')
time.sleep(1)
pyautogui.press('up')
while True:
img = np.array(sct.grab(monitor))
obstacles = img[ROI[0][0]:ROI[0][1], ROI[1][0]:ROI[1][1]]
# obstacles = img[43:256, 950:1300]
gray = cv2.cvtColor(obstacles, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(gray, 90, 255, cv2.THRESH_BINARY)
edged = cv2.Canny(thresh, 30, 200)
contours, hierarchy = cv2.findContours(edged, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
# cv2.drawContours(out, contours, -1, (0, 0, 255), 2)
points = findBoundingBoxesWithShift(contours)
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
dino_coords = find.find_dino(img)
# print(dist)
# print(start_point, end_point)
# if dist > 20 and dist < 260:
if (len(points) >= 9 and len(points) < 20) or force_gameover:
# Run tesseract OCR on image
gameover = img[gameover_ROI[0][0]:gameover_ROI[0][1],
gameover_ROI[1][0]:gameover_ROI[1][1]]
text = pytesseract.image_to_string(gameover, config=tess_config)
# print('maybe gameover')
if (text.lower().replace(' ', '') == "gameover") or force_gameover:
score_img = img[score_ROI[0][0]:score_ROI[0][1],
score_ROI[1][0]:score_ROI[1][1]]
# gray_score = cv2.cvtColor(score_img, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(score_img, 90, 255,
cv2.THRESH_BINARY)
score = ocr.get_score(thresh)
time_score = time.time() - score_time
try:
print(
f'Game Over! Game: {game} - Score: {int(score)}, Time Score: {time_score}',
end='\n\n')
except:
print(f"{score} is not an integer")
if time_score >= 3:
game += 1
break
if score_time - time.time() > 400:
force_gameover = True
def play(net):
global frame, ROI, play_toggle, game, score_ROI, generation, max_fitness, winner
print('playing')
last_dist = 0
tess_config = ('-l eng --oem 1 --psm 3')
gameover_ROI = [(50, 120), (300, 900)]
score_time = time.time()
force_gameover = False
generation += 1
fgo_thresh = 350 + generation * 50
pyautogui.keyDown('command')
pyautogui.press('r')
pyautogui.keyUp('command')
print('reloaded page')
time.sleep(1)
scroll_go = False
force_gameover = False
score_time = time.time()
# start game
pyautogui.press('up')
time.sleep(1)
pyautogui.press('up')
while True:
# Get raw pixels from the screen, save it to a Numpy array
try:
# screen shot
img = np.array(sct.grab(monitor))
# roi of just obstacles
obstacles = img[ROI[0][0]:ROI[0][1], ROI[1][0]:ROI[1][1]]
# obstacles = img[43:256, 950:1300]
gray = cv2.cvtColor(obstacles, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(gray,90,255,cv2.THRESH_BINARY) # get binary image of abstacles
edged = cv2.Canny(thresh, 30, 200) # edge detection to find obstacle hitboxes
# find contours of obstacles
contours, hierarchy = cv2.findContours(edged, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
# get hitboxes
points = findBoundingBoxesWithShift(contours)
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
dino_coords = find.find_dino(img)
# find dist between dino and nearest obstacle
dist, start_point, end_point, width = findDistance(dino_coords, points)
# feed data to nn and run output
dino_y = start_point[-1]
cac_y = end_point[-1]
delta_dist = abs(last_dist - dist)
output = net.activate([dist, cac_y, dino_y, width, delta_dist])
# [0] jump, [1] crouch, [2] idle
if output[0] >= output[1] and output[0] >= output[2]: # if jump is the highest value
pyautogui.press('up')
elif output[1] >= output[0] and output[1] >= output[2]:
pyautogui.press('down')
if cac_y > 310:
genome.fitness += 150
elif output[2] >= output[0] and output[2] >= output[1]:
pass
# check if game ended
if (len(points) >= 9 and len(points) < 20 and last_dist == dist) or force_gameover:
# Run tesseract OCR on image to see if words "game over" are on screen
gameover = img[gameover_ROI[0][0]:gameover_ROI[0][1], gameover_ROI[1][0]:gameover_ROI[1][1]]
text = pytesseract.image_to_string(gameover, config=tess_config)
# print('maybe gameover')
if (text.lower().replace(' ', '') == "gameover") or force_gameover:
# for when nn presses down and scrolls under game over text
if scroll_go:
pyautogui.scroll(20, x=690, y=450)
time.sleep(1)
pyautogui.scroll(20, x=690, y=450)
print('scrolling')
time.sleep(0.3)
# total time the dino has ran for
time_score = time.time() - score_time
# make sure score isnt ''
score = ''
count = 0
while score == '' and count < 5:
img = np.array(sct.grab(monitor))
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
score_img = img[score_ROI[0][0]:score_ROI[0][1], score_ROI[1][0]:score_ROI[1][1]]
# gray_score = cv2.cvtColor(score_img, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(score_img,90,255,cv2.THRESH_BINARY)
score = ocr.get_score(thresh)
time.sleep(0.1)
count += 1
if count >= 4:
score = 44
print(f'Bonus: {genome.fitness}')
try:
print(f'Game Over! Game: {game} - Score: {int(score)}, Time Score: {time_score}', end='\n\n')
except:
print(f"{score} is not an integer")
# print(f'Upper: {time_score * ((int(score)/100)+20)}, Lower: {time_score * 5}', end='\n\n')
if time_score < 1800:
if int(score) > time_score * ((int(score)/100)+13):
score = (time_score - 2) * 10
print(f'Adjusted Score: {score}')
elif int(score) < time_score * 5:
score = (time_score) * 8
print(f'Adjusted Score: {score}')
# make sure game lasted over three seconds
if time_score >= 3:
game += 1
if not force_gameover or scroll_go:
# set fitness
genome.fitness += int(score)
if genome.fitness > max_fitness:
winner = genome
break
else:
# replay game
reload()
pyautogui.scroll(20, x=690, y=450)
pyautogui.press('up')
time.sleep(1)
pyautogui.press('up')
score_time = time.time()
force_gameover = False
if len(points) > 25: # if nn spams down and scrolls the page
# pyautogui.scroll(20, x=690, y=450)
force_gameover = True
scroll_go = True
print('scroll gameover')
time.sleep(0.5)
# when game cant see the game over test, it times out
if score_time - time.time() > fgo_thresh:
print('timeout')
force_gameover = True
if int(time.time())%1000 == 0:
pyautogui.scroll(20, x=690, y=450)
last_dist = dist
except KeyboardInterrupt:
print('program terminated: keyboard interrupt')
cv2.destroyAllWindows()
break
# obstacles = img[43:256, 950:1300]
gray = cv2.cvtColor(obstacles, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(gray,90,255,cv2.THRESH_BINARY)
out = obstacles.copy()
edged = cv2.Canny(thresh, 30, 200)
contours, hierarchy = cv2.findContours(edged, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
# cv2.drawContours(out, contours, -1, (0, 0, 255), 2)
points = findBoundingBoxesWithShift(contours)
out = drawBoundingBoxes(img.copy(), points, (0, 0, 255))
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
if frame % update_dino == 0:
dino_coords = find.find_dino(img)
dist, start_point, end_point = findDistance(dino_coords, points)
# print(dist)
# print(start_point, end_point)
if frame % control_frame_interval == 0:
if dist > 20 and dist < 250:
pyautogui.press('up')
out = cv2.line(out, start_point, end_point, (0, 0, 0), 2)
cv2.rectangle(out, dino_coords[0][0], dino_coords[0][1], (0, 255, 0), 2)
# cv2.imshow(title, screen)
cv2.imshow('processed', out)
def show_vision():
global fps, start_time, score_ROI
x = True
font = cv2.FONT_HERSHEY_SIMPLEX
while True:
# Get raw pixels from the screen, save it to a Numpy array
try:
img = np.array(sct.grab(monitor))
obstacles = img[ROI[0][0]:ROI[0][1], ROI[1][0]:ROI[1][1]]
# obstacles = img[43:256, 950:1300]
gray = cv2.cvtColor(obstacles, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(gray, 90, 255, cv2.THRESH_BINARY)
out = obstacles.copy()
edged = cv2.Canny(thresh, 30, 200)
contours, hierarchy = cv2.findContours(edged, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
# cv2.drawContours(out, contours, -1, (0, 0, 255), 2)
points = findBoundingBoxesWithShift(contours)
out = drawBoundingBoxes(img.copy(), points, (0, 0, 255))
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
dino_coords = find.find_dino(img)
dist, start_point, end_point = findDistance(dino_coords, points)
# print(dist)
# print(start_point, end_point)
out = cv2.line(out, start_point, end_point, (0, 0, 0), 2)
cv2.rectangle(out, dino_coords[0][0], dino_coords[0][1],
(0, 255, 0), 2)
if not play_game.isAlive() and x:
print('not playing!')
x = False
fps += 1
TIME = time.time() - start_time
cv2.putText(out, f'FPS: {int((fps / TIME)*1000)/1000}', (0, 25),
font, 1, (0, 0, 0), 2, cv2.LINE_AA)
if (TIME) >= display_time:
fps = 0
start_time = time.time()
# cv2.imshow(title, screen)
score_img = img[score_ROI[0][0]:score_ROI[0][1],
score_ROI[1][0]:score_ROI[1][1]]
# gray_score = cv2.cvtColor(score_img, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(score_img, 90, 255, cv2.THRESH_BINARY)
cv2.imshow('Score', thresh)
cv2.imshow('Computer Vision', out)
# Press "q" to quit
if cv2.waitKey(25) & 0xFF == ord(
"q"): # take only the lasy byte and compare
cv2.destroyAllWindows()
break
except KeyboardInterrupt:
print('program terminated')
cv2.destroyAllWindows()
break
subprocess.call('clear')
def play():
global frame, ROI, control_frame_interval, play_toggle, game, score_ROI
print('playing')
last_dist = 0
config = ('-l eng --oem 1 --psm 3')
gameover_ROI = [(50, 120), (300, 900)]
score_time = time.time()
force_gameover = False
while True:
# Get raw pixels from the screen, save it to a Numpy array
try:
img = np.array(sct.grab(monitor))
obstacles = img[ROI[0][0]:ROI[0][1], ROI[1][0]:ROI[1][1]]
# obstacles = img[43:256, 950:1300]
gray = cv2.cvtColor(obstacles, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(gray, 90, 255, cv2.THRESH_BINARY)
edged = cv2.Canny(thresh, 30, 200)
contours, hierarchy = cv2.findContours(edged, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
# cv2.drawContours(out, contours, -1, (0, 0, 255), 2)
points = findBoundingBoxesWithShift(contours)
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
dino_coords = find.find_dino(img)
dist, start_point, end_point = findDistance(dino_coords, points)
# print(dist)
# print(start_point, end_point)
if frame % control_frame_interval == 0:
if dist > 20 and dist < 260:
key.press('up')
if (len(points) >= 9 and len(points) < 20
and last_dist == dist) or force_gameover:
# Run tesseract OCR on image
gameover = img[gameover_ROI[0][0]:gameover_ROI[0][1],
gameover_ROI[1][0]:gameover_ROI[1][1]]
text = pytesseract.image_to_string(gameover, config=config)
# print('maybe gameover')
if (text.lower().replace(' ', '')
== "gameover") or force_gameover:
key.press('up')
score_img = img[score_ROI[0][0]:score_ROI[0][1],
score_ROI[1][0]:score_ROI[1][1]]
# gray_score = cv2.cvtColor(score_img, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(score_img, 90, 255,
cv2.THRESH_BINARY)
score = str(
pytesseract.image_to_string(thresh, config='digits'))
time_score = time.time() - score_time
try:
print(
f'Game Over! Game: {game} - Score: {int(score)}, Time Score: {time_score}',
end='\n\n')
except:
print(f"{score} is not an integer")
scores.append(score)
if game >= games:
# raise KeyboardInterrupt
pass
if time_score >= 3:
game += 1
force_gameover = False
time.sleep(1)
key.press('up')
score_time = time.time()
if score_time - time.time() > 250:
force_gameover = True
frame += 1
last_dist = dist
# print('playing')
if frame > 999999:
frame = 0
except KeyboardInterrupt:
print('program terminated')
cv2.destroyAllWindows()
break
def show_vision():
global fps, start_time, score_ROI, winner, stats, config
x = True
font = cv2.FONT_HERSHEY_SIMPLEX
while True:
# Get raw pixels from the screen, save it to a Numpy array
try:
# get screen
img = np.array(sct.grab(monitor))
obstacles = img[ROI[0][0]:ROI[0][1], ROI[1][0]:ROI[1][1]]
gray = cv2.cvtColor(obstacles, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(gray, 90, 255, cv2.THRESH_BINARY)
out = obstacles.copy()
# edge detetection
edged = cv2.Canny(thresh, 30, 200)
contours, hierarchy = cv2.findContours(edged, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
# find & draw hitboxes
points = findBoundingBoxesWithShift(contours)
out = drawBoundingBoxes(img.copy(), points, (0, 0, 255))
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
dino_coords = find.find_dino(img)
dist, start_point, end_point, width = findDistance(
dino_coords, points)
# draw line from dino to nearest obstacle
out = cv2.line(out, start_point, end_point, (0, 0, 0), 2)
cv2.rectangle(out, dino_coords[0][0], dino_coords[0][1],
(0, 255, 0), 2)
# when error is thrown in NEAT, tell user
if not NEAT.isAlive() and x:
print('not playing!')
x = False
fps += 1
TIME = time.time() - start_time
# add fps top left
cv2.putText(out, f'FPS: {int((fps / TIME)*1000)/1000}', (0, 25),
font, 1, (0, 0, 0), 2, cv2.LINE_AA)
if (TIME) >= display_time:
fps = 0
start_time = time.time()
score_img = img[score_ROI[0][0]:score_ROI[0][1],
score_ROI[1][0]:score_ROI[1][1]]
ret, thresh = cv2.threshold(score_img, 90, 255, cv2.THRESH_BINARY)
# show frames
cv2.imshow('Score', thresh)
cv2.imshow('Computer Vision', out)
# Press "q" to quit
# if winner != '':
# raise KeyboardInterrupt
if cv2.waitKey(25) & 0xFF == ord(
"q"): # take only the lasy byte and compare
cv2.destroyAllWindows()
break
except KeyboardInterrupt:
print('program terminated')
cv2.destroyAllWindows()
with open('winner-ctrnn', 'wb') as f:
pickle.dump(winner, f)
visualize.plot_stats(stats,
ylog=True,
view=True,
filename="ctrnn-fitness.svg")
try:
visualize.plot_species(stats,
view=True,
filename="ctrnn-speciation.svg")
except ValueError:
print('An entire generation has not passed yet')
if winner != '':
node_names = {
-1: 'distance',
-2: 'obstacle y',
-3: 'dino y',
-4: 'width',
-5: 'speed',
-5: 'dtime',
0: 'jump',
1: 'crouch',
2: 'idle'
}
visualize.draw_net(config, winner, True, node_names=node_names)
visualize.draw_net(config,
winner,
view=True,
node_names=node_names,
filename="winner-ctrnn.gv")
visualize.draw_net(config,
winner,
view=True,
node_names=node_names,
filename="winner-ctrnn-enabled.gv",
show_disabled=False)
visualize.draw_net(config,
winner,
view=True,
node_names=node_names,
filename="winner-ctrnn-enabled-pruned.gv",
show_disabled=False,
prune_unused=True)
break
except:
with open('winner-ctrnn-new', 'wb') as f:
pickle.dump(winner, f)
visualize.plot_stats(stats,
ylog=True,
view=True,
filename="ctrnn-fitness.svg")
try:
visualize.plot_species(stats,
view=True,
filename="ctrnn-speciation.svg")
except ValueError:
print('An entire generation has not passed yet')
if winner != '':
node_names = {
-1: 'distance',
-2: 'obstacle y',
-3: 'dino y',
-4: 'dtime',
0: 'jump',
1: 'crouch',
2: 'idle'
}
visualize.draw_net(config, winner, True, node_names=node_names)
visualize.draw_net(config,
winner,
view=True,
node_names=node_names,
filename="winner-ctrnn.gv")
visualize.draw_net(config,
winner,
view=True,
node_names=node_names,
filename="winner-ctrnn-enabled.gv",
show_disabled=False)
visualize.draw_net(config,
winner,
view=True,
node_names=node_names,
filename="winner-ctrnn-enabled-pruned.gv",
show_disabled=False,
prune_unused=True)
