def run(self):
self.isActive = True
for module in self.modules:
module.reset()
window = GameWindow("BLACK DESERT")
window.move_to_foreground()
point = window.rect.center()
pyautogui.moveTo(point[0], point[1], duration=2)
while self.isActive:
# grab frame
frame = window.grab_frame()
# fill up queue
for module in self.modules:
if module.isEnabled:
for action in module.getActions(frame):
self.actions.put(action)
# process queue
while not self.actions.empty():
self.actions.get().execute()
if not self.isActive:
break
print("sleeping...")
for i in range(20):
if not self.isActive:
break
time.sleep(0.1)
print('main thread stopped')
def run(self):
self.reel_state = 2
self.window = GameWindow("BLACK DESERT")
self.window.move_to_foreground()
self.prev_time = None
point = self.window.rect.center()
pyautogui.moveTo(point[0], point[1], duration=2)
comp_path = "resources/fishing_space_bar.jpg"
self.comp_img = cv.imread(comp_path, 0)
while 1:
state = self.getState()
if (actions):
if (state == 1):
self.actions(1)
# break
elif (state == 2):
self.actions(2)
else:
self.stream_img = self.window.grab_frame()
stream_gray = cv.cvtColor(self.stream_img, cv.COLOR_BGR2GRAY)
method = cv.TM_SQDIFF
result = cv.matchTemplate(self.comp_img, stream_gray, method)
mn, _, mnLoc, _ = cv.minMaxLoc(result)
MPx, MPy = mnLoc
trows, tcols = self.comp_img.shape[:2]
font = cv.FONT_HERSHEY_SIMPLEX
if (state == 0):
# idle
cv.putText(self.stream_img, "Idle Detected", (180, 25),
font, 0.8, (255, 0, 0), 2, cv.LINE_AA)
elif (state == 1):
# space bar
cv.putText(self.stream_img, "Space Bar Detected",
(180, 25), font, 0.8, (255, 0, 0), 2,
cv.LINE_AA)
print(str(MPx) + " " + str(MPy))
cv.rectangle(self.stream_img, (MPx, MPy),
(MPx + tcols, MPy + trows), (0, 0, 255), 2)
cv.imshow("output", self.stream_img)
cv.waitKey(0)
cv.destroyAllWindows()
print("sleeping...")
time.sleep(2)
print("leaving")
def run(self):
window = GameWindow("BLACK DESERT")
window.move_to_foreground()
# take screenshot here
print("taking screenshot")
screenshot_dir = "screenshots"
if not os.path.exists(screenshot_dir):
os.makedirs(screenshot_dir)
file_name = str(uuid.uuid4()) + ".jpg"
cv.imwrite(os.path.join(screenshot_dir, file_name),
window.grab_frame())
def run(self):
window = GameWindow("BLACK DESERT")
window.move_to_foreground()
point = window.rect.center()
pyautogui.moveTo(point[0], point[1], duration=2)
while 1:
direct_input.PressKey("R")
time.sleep(1)
direct_input.ReleaseKey("R")
time.sleep(1)
def _find_space_y_button_handler(self):
comp_path = "resources/fishing_space_bar.jpg"
comp_img = cv.imread(comp_path, 0)
window = GameWindow("BLACK DESERT")
window.move_to_foreground()
for i in range(20):
img = window.grab_frame()
img_gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
method = cv.TM_SQDIFF
result = cv.matchTemplate(comp_img, img_gray, method)
mn, _, mnLoc, _ = cv.minMaxLoc(result)
MPx, MPy = mnLoc
print("x: " + str(MPx) + ", y: " + str(MPy))
time.sleep(0.1)
def __init__(self):
self.isEnabled = False
self.reelInFishAction = Action(0, self.reelInFish)
self.castLineAction = Action(0, self.castLine)
self.classifier = Classifier('./ml/')
self.comp_img = cv.imread('resources/fishing_space_bar.jpg', 0)
self.window = GameWindow("BLACK DESERT")
self.colorGrabber = ColorGrabber()
self.outputColors = False
self.outputKeys = False
self.swapRods = False
self.rodCharSeq = ''
self.rodSwapThresh = 5
self.collectAll = True
self.discardBlue = False
self.discardGreen = False
self.collectUnknowns = False
self.reset()
def main():
window = GameWindow("BLACK DESERT")
window.move_to_foreground()
point = window.local_to_global((720, 360))
pyautogui.moveTo(point[0], point[1], duration=2)
#time.sleep(2)
#pyautogui.mouseDown(button='right')
#time.sleep(2)
#pyautogui.mouseUp(button='right')
cut_tree()
while True:
cv.imshow("Window", window.grab_frame())
if cv.waitKey(1) == ord('q'):
break
class StreamThread(QThread):
reel_state = 2
def __init__(self):
super(StreamThread, self).__init__()
self.keySequenceDetector = KeySequenceDetector()
self.keySequenceDetector.output = True
self.classifier = Classifier('./ml/')
def __del__(self):
self.wait()
def getState(self): # idle=0, start=1, catch=2
self.window.move_to_foreground()
self.stream_img = self.window.grab_frame()
stream_gray = cv.cvtColor(self.stream_img, cv.COLOR_BGR2GRAY)
method = cv.TM_SQDIFF
result = cv.matchTemplate(self.comp_img, stream_gray, method)
mn, _, mnLoc, _ = cv.minMaxLoc(result)
MPx, MPy = mnLoc
print("x: " + str(MPx) + ", y: " + str(MPy))
status = 0
if (MPx > start_target_x - target_thresh
and MPx < start_target_x + target_thresh
and MPy > start_target_y - target_thresh
and MPy < start_target_y + target_thresh):
if (self.reel_state == 2):
self.reel_state = 1
spaces_folder = spaces_folder_cast
else:
self.reel_state = 2
spaces_folder = spaces_folder_reel
if output_spaces:
if not os.path.exists(spaces_folder):
os.makedirs(spaces_folder)
for i in range(10):
pyautogui.move(random.gauss(0, 50),
random.gauss(0, 5),
duration=random.gauss(0.1, .01))
cv.imwrite(
os.path.join(spaces_folder,
str(uuid.uuid4()) + '.jpg'),
self.window.grab_frame())
return self.reel_state
elif (MPx > catch_target_x - target_thresh
and MPx < catch_target_x + target_thresh
and MPy > catch_target_y - target_thresh
and MPy < catch_target_y + target_thresh):
return 2
else:
return 0
def actions(self, action):
if (action == 1):
if output_result:
if not os.path.exists(results_folder):
os.makedirs(results_folder)
if not self.prev_time is None:
elap_time = time.time() - self.prev_time
with open(results_folder + '/minigame_times.txt',
'a+') as f:
f.write(f'{elap_time}\n')
self.prev_time = time.time()
PressKey("SPACE")
print("casting line")
ReleaseKey("SPACE")
time.sleep(5)
# print("moving camera")
# pyautogui.move(random.gauss(0, 600), random.gauss(0, 40), duration=random.gauss(2, .1))
if output_empty:
if not os.path.exists(no_spaces_folder):
os.makedirs(no_spaces_folder)
for i in range(10):
pyautogui.move(random.gauss(0, 50),
random.gauss(0, 5),
duration=random.gauss(0.1, .01))
cv.imwrite(
os.path.join(no_spaces_folder,
str(uuid.uuid4()) + '.jpg'),
self.window.grab_frame())
elif (action == 2):
PressKey("SPACE")
print("reeling in")
ReleaseKey("SPACE")
time.sleep(1.6 + 0.04)
PressKey("SPACE")
print("playing game")
ReleaseKey("SPACE")
time.sleep(3.0)
img = self.window.grab_frame()
now = time.time()
keySequence = self.classifier.evaluate_and_save(
img, './ml/new_ui_keys/raw')
print(f'Time to read seq: {time.time()-now}')
if keySequence is None:
keySequence = []
# self.keySequenceDetector.processFrames(2, 2)
# keySequence = self.keySequenceDetector.getKeySequence()
print("keys: " + str([key_names[key] for key in keySequence]))
# time.sleep(1.2)
# if (len(keySequence) == 0):
# print("writing frame to file as failure")
# self.stream_img = self.window.grab_frame()
# if not os.path.exists(output_folder):
# os.makedirs(output_folder)
# file_name = str(uuid.uuid4()) + ".jpg"
# cv.imwrite(os.path.join(output_folder, file_name), self.stream_img)
if press_keys:
for key in keySequence:
self.tapKey(key)
if output_result:
if not os.path.exists(results_folder):
os.makedirs(results_folder)
img = cv.putText(img,
str([key_names[key] for key in keySequence]),
(400, 245), cv.FONT_HERSHEY_SIMPLEX, 0.8,
(0, 255, 0), 2)
cv.imwrite(os.path.join(results_folder, f'{uuid.uuid4()}.jpg'),
img)
if not self.prev_time is None:
elap_time = time.time() - self.prev_time
with open(results_folder + '/reeling_times.txt',
'a+') as f:
f.write(f'{elap_time}\n')
self.prev_time = time.time()
time.sleep(6)
if output_catch:
if not os.path.exists(catch_folder):
os.makedirs(catch_folder)
cv.imwrite(f'{catch_folder}/{uuid.uuid4()}.jpg',
self.window.grab_frame())
self.tapKey(4)
def sleep(self):
sleep_time = 0.03 + np.clip(random.gauss(0.04, 0.015), 0, 0.8)
time.sleep(sleep_time)
def tapKey(self, key):
key_string = ""
if key == 0:
key_string = "W"
elif key == 1:
key_string = "A"
elif key == 2:
key_string = "S"
elif key == 3:
key_string = "D"
else:
key_string = "R"
PressKey(key_string)
self.sleep()
ReleaseKey(key_string)
self.sleep()
def run(self):
self.reel_state = 2
self.window = GameWindow("BLACK DESERT")
self.window.move_to_foreground()
self.prev_time = None
point = self.window.rect.center()
pyautogui.moveTo(point[0], point[1], duration=2)
comp_path = "resources/fishing_space_bar.jpg"
self.comp_img = cv.imread(comp_path, 0)
while 1:
state = self.getState()
if (actions):
if (state == 1):
self.actions(1)
# break
elif (state == 2):
self.actions(2)
else:
self.stream_img = self.window.grab_frame()
stream_gray = cv.cvtColor(self.stream_img, cv.COLOR_BGR2GRAY)
method = cv.TM_SQDIFF
result = cv.matchTemplate(self.comp_img, stream_gray, method)
mn, _, mnLoc, _ = cv.minMaxLoc(result)
MPx, MPy = mnLoc
trows, tcols = self.comp_img.shape[:2]
font = cv.FONT_HERSHEY_SIMPLEX
if (state == 0):
# idle
cv.putText(self.stream_img, "Idle Detected", (180, 25),
font, 0.8, (255, 0, 0), 2, cv.LINE_AA)
elif (state == 1):
# space bar
cv.putText(self.stream_img, "Space Bar Detected",
(180, 25), font, 0.8, (255, 0, 0), 2,
cv.LINE_AA)
print(str(MPx) + " " + str(MPy))
cv.rectangle(self.stream_img, (MPx, MPy),
(MPx + tcols, MPy + trows), (0, 0, 255), 2)
cv.imshow("output", self.stream_img)
cv.waitKey(0)
cv.destroyAllWindows()
print("sleeping...")
time.sleep(2)
print("leaving")
class AutoFishingModule():
space_icon_x = 588
space_icon_ys = 197, 192
space_icon_thresh = 2
key_names = 'W', 'A', 'S', 'D'
def __init__(self):
self.isEnabled = False
self.reelInFishAction = Action(0, self.reelInFish)
self.castLineAction = Action(0, self.castLine)
self.classifier = Classifier('./ml/')
self.comp_img = cv.imread('resources/fishing_space_bar.jpg', 0)
self.window = GameWindow("BLACK DESERT")
self.colorGrabber = ColorGrabber()
self.outputColors = False
self.outputKeys = False
self.swapRods = False
self.rodCharSeq = ''
self.rodSwapThresh = 5
self.collectAll = True
self.discardBlue = False
self.discardGreen = False
self.collectUnknowns = False
self.reset()
def reset(self):
self.last_frame = None
self.state = State.LINE_OUT_OF_WATER
self.rodCatchCount = 0
def getActions(self, frame):
frame_gray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY)
method = cv.TM_SQDIFF
result = cv.matchTemplate(self.comp_img, frame_gray, method)
mn, _, mnLoc, _ = cv.minMaxLoc(result)
MPx, MPy = mnLoc
print("x: " + str(MPx) + ", y: " + str(MPy))
if (abs(MPx - self.space_icon_x) < self.space_icon_thresh and (abs(MPy - space_icon_y) < self.space_icon_thresh for space_icon_y in self.space_icon_ys)):
if (self.state == State.LINE_OUT_OF_WATER):
return [self.castLineAction,]
else:
return [self.reelInFishAction,]
return []
def reelInFish(self):
PressKey("SPACE")
print("reeling in")
ReleaseKey("SPACE")
time.sleep(1.6 + 0.04)
PressKey("SPACE")
print("playing game")
ReleaseKey("SPACE")
time.sleep(3.0)
img = self.window.grab_frame()
if self.outputKeys:
keySequence = self.classifier.evaluate_and_save(img, './ml/new_ui_keys/raw')
else:
keySequence = self.classifier.evaluate(img)
if keySequence is None:
keySequence = []
print("keys: " + str([self.key_names[key] for key in keySequence]))
for key in keySequence:
self.tapKey(key)
time.sleep(6)
# decide wether to grab or not
if self.collectAll:
print('shortcutted')
self.tapKey(4)
else:
img = self.window.grab_frame()
if self.outputColors:
folder = './ml/new_ui_keys/catches'
if not os.path.exists(folder):
os.makedirs(folder)
cv.imwrite(f'{folder}/{uuid.uuid4()}.tiff', img)
if self.colorGrabber.evaluate(img, self.discardBlue, self.discardGreen, self.collectUnknowns):
self.tapKey(4)
self.rodCatchCount += 1
if self.swapRods and self.rodCatchCount >= self.rodSwapThresh:
print('cycling rods')
time.sleep(0.5)
for char in self.rodCharSeq:
self.tapChar(char)
self.rodCatchCount = 0
self.state = State.LINE_OUT_OF_WATER
def castLine(self):
PressKey("SPACE")
print("casting line")
ReleaseKey("SPACE")
time.sleep(5)
self.state = State.LINE_IN_WATER
def key_sleep(self):
sleep_time = 0.04 + np.clip(random.gauss(0.04, 0.015), 0, 0.8)
time.sleep(sleep_time)
def tapChar(self, char):
PressKey(char)
self.key_sleep()
ReleaseKey(char)
self.key_sleep()
def tapKey(self, key):
key_string = ""
if key == 0:
key_string = "W"
elif key == 1:
key_string = "A"
elif key == 2:
key_string = "S"
elif key == 3:
key_string = "D"
else:
key_string = "R"
PressKey(key_string)
self.key_sleep()
ReleaseKey(key_string)
self.key_sleep()
import os, sys, time
import cv2 as cv
import pyautogui
my_dir = os.path.dirname(sys.argv[0])
sys.path.append(os.path.join(my_dir, '../'))
from utils.game_window import GameWindow
start_target_x = 588
start_target_y = 197
target_thresh = 5
comp_path = "../resources/fishing_space_bar.jpg"
comp_img = cv.imread(comp_path, 0)
window = GameWindow("BLACK DESERT")
window.move_to_foreground()
point = window.rect.center()
pyautogui.moveTo(point[0], point[1], duration=2)
seen_thresh = 100
enter_time = time.time()
last_seen = time.time()
finish_time = time.time()
frames_since_seen = seen_thresh + 1
while True:
img = window.grab_frame()
stream_gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
method = cv.TM_SQDIFF
result = cv.matchTemplate(comp_img, stream_gray, method)
def processFrames(self, pos=1, read_time=1):
window = GameWindow("BLACK DESERT")
window.move_to_foreground()
print("grabbing frames for " + str(read_time) + " seconds")
mask_path = "resources/combined_key_color.tiff"
mask = cv.imread(mask_path, 1)
show_amnt = 6
key_detectors = []
for i in range(pos):
key_detectors.append(KeyDetectorDiff(mask, i))
cut_time = time.time() + read_time
f = 0
frames = []
while time.time() <= cut_time:
f += 1
print("procressing frame " + str(f))
color_frame = window.grab_frame()
frame = cv.cvtColor(color_frame, cv.COLOR_BGR2GRAY)
frames.append(color_frame)
for key_detector in key_detectors:
key_detector.processFrame(frame)
comb_frame = np.zeros((KeyDetectorDiff.dy, KeyDetectorDiff.dx),
np.int16)
for key_detector in key_detectors:
comb_frame += key_detector.getDifferenceImage()
_, max_x, max_y = KeyDetectorDiff.getMaxAndPos(comb_frame)
self.start_x, self.start_y = key_detectors[0].x + max_x, key_detectors[
0].y + max_y
self.w, self.h = mask.shape[1], mask.shape[0]
self.last_frame = color_frame
if self.output:
if not os.path.exists(output_folder):
os.makedirs(output_folder)
out_dir = output_folder + str(uuid.uuid4())
print("saving to " + out_dir)
os.makedirs(out_dir)
for i in range(1): #len(frames)):
cv.imwrite(os.path.join(out_dir, "frame" + str(i) + ".jpg"),
frames[i])
di = KeyDetectorDiff.di
max, max_x, max_y = KeyDetectorDiff.getMaxAndPos(comb_frame)
keySequence = self.getKeySequence()
with open(out_dir + "/" + csv_name, mode="w") as csv_file:
csv_writer = csv.writer(csv_file,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow([max_x, max_y, keySequence])
if self.show_image:
di = KeyDetectorDiff.di
mask_red = mask[:, :, 2] > pixel_thresh
h, w = mask.shape[0], mask.shape[1]
for c in range(show_amnt):
for i in range(h):
for j in range(w):
if self.show_overlay and mask_red[i, j] > 0:
color_frame[key_detectors[0].y + i + max_y,
key_detectors[0].x + j + max_x +
(c * di)] = [255, 0, 0]
if self.show_border and (i < 2 or i >= h - 2 or j < 2
or j >= w - 2):
color_frame[key_detectors[0].y + i + max_y,
key_detectors[0].x + j + max_x +
35 * c] = [0, 0, 255]
cv.imshow("frame", color_frame)
cv.waitKey(0)
cv.destroyAllWindows()