def sortDeque(d):
copy = d.copy()
l = deque()
min = KeyEvent(keyboard.Key.enter, 999999999999, 99999999999)
while len(copy) > 0:
for i in copy:
if i.time < min.time:
min = i
l.append(min)
copy.remove(min)
min = KeyEvent(keyboard.Key.enter, 999999999999, 99999999999)
return l
def on_release(key):
global running, quit_counter, shift_modifier, ctrl_modifier, alt_modifier, altgr_modifier, fn_modifier, cmd_modifier
if hasattr(key, "name"):
name = key.name
elif hasattr(key, "char"):
name = key.char
else:
name = str(key)
if key == keyboard.Key.shift:
shift_modifier = 0
if key == keyboard.Key.ctrl:
ctrl_modifier = 0
if key == keyboard.Key.alt:
alt_modifier = 0
if key == keyboard.Key.alt_gr:
altgr_modifier = 0
if key == keyboard.Key.cmd:
cmd_modifier = 0
add_event(KeyEvent(time.time(), name, pressed=False, shift=shift_modifier, ctrl=ctrl_modifier, alt=alt_modifier,
altgr=altgr_modifier, fn=fn_modifier, cmd=cmd_modifier))
if key == keyboard.Key.esc:
quit_counter -= 1
if quit_counter == 0:
running = False
print("[Manual] Killing tester.")
def get_key(self) -> KeyEvent:
attr_o = termios.tcgetattr(_STDIN_FD)
termios.tcsetattr(_STDIN_FD, termios.TCSANOW, _ATTR_NEW)
buf = os.read(_STDIN_FD, _READ_BUFFER)
kname = self.__keymap.get(buf, 'UNKNOWN_KEY')
if kname == 'UNKNOWN_KEY' and buf in keys.K_MAP.keys():
buf = keys.K_MAP.get(buf, buf)
kname = self.__keymap.get(buf, 'UNKNOWN_KEY')
termios.tcsetattr(_STDIN_FD, termios.TCSANOW, attr_o)
return KeyEvent(buf, kname)
def __read_key_loop(self):
while True:
if self.__stop:
continue
k = os.read(_STDIN_FD, _READ_BUFFER)
kname = self.__keymap.get(k, "UNKNOWN_KEY")
if k == keys.KEY_RETURN:
k = keys.KEY_ENTER
evt = KeyEvent(k, kname)
self.__queue.append(evt)
def on_release(key):
try:
if not record:
return
t1 = keyTimes.get(key) - t0
dt = time.time() - keyTimes.get(key)
keyEventDeque.append(KeyEvent(key, t1, dt))
keyTimes.update({key: 0})
print('{0} released'.format(key))
if key == keyboard.Key.esc:
newDeque = sortDeque(keyEventDeque)
for i in newDeque:
print(i)
pickle.dump(newDeque, open("sequence.p", "wb"))
# Stop listener
return False
except Exception as e:
print("Error", e)
def on_press(key):
global shift_modifier, ctrl_modifier, alt_modifier, altgr_modifier, fn_modifier, cmd_modifier
name = ""
if hasattr(key, "name"):
name = key.name
elif hasattr(key, "char"):
name = key.char
else:
name = str(key)
if key == keyboard.Key.shift:
shift_modifier = 1
if key == keyboard.Key.ctrl:
ctrl_modifier = 1
if key == keyboard.Key.alt:
alt_modifier = 1
if key == keyboard.Key.alt_gr:
altgr_modifier = 1
if key == keyboard.Key.cmd:
cmd_modifier = 1
add_event(KeyEvent(time.time(), name, pressed=True, shift=shift_modifier, ctrl=ctrl_modifier, alt=alt_modifier,
altgr=altgr_modifier, fn=fn_modifier, cmd=cmd_modifier))
def perform_interaction(best_box,
app_x,
app_y,
app_w,
app_h,
input_string=generate_input_string()):
x_mod = (0.5 - random.random()) * best_box[3]
y_mod = (0.5 - random.random()) * best_box[4]
x = int(
max(app_x, min(app_x + app_w,
app_x + ((best_box[1] + x_mod) * app_w))))
y = int(
max(app_y, min(app_y + app_h,
app_y + ((best_box[2] + y_mod) * app_h))))
# y_start = max(app_y, min(app_y + height - 5, app_y + int(height*(best_box[2] - best_box[4]/2))-5))
# y_end = max(app_y+5, min(app_y + height, app_y + int(height*(best_box[2]+best_box[4]/2))+5))
#
# x_start = max(app_x, min(app_x + width - 5, app_x + int(width*(best_box[1]-best_box[3]/2))-5))
# x_end = max(app_x + 5, min(app_x + width, app_x + int(width*(best_box[1]+best_box[3]/2))+5))
#
# image_clicked = raw_image[y_start:y_end,
# x_start:x_end]
#
# output_img = output_dir + "/images/" + str(event)
#
# np.save(output_img, image_clicked)
#
# cv2.imwrite(output_img + ".jpg", image_clicked)
#
# widget = yolo.names[int(best_box[0 ])]
#
# interactions.append([widget, event, input_string])
#
#
# with open(test_file, "a+") as t_f:
# t_f.write(widget + "," + str(event) + "\n")
#
# pyautogui.moveTo(x, y)
#
# event = event + 1
#
# if widget == "button" or widget == "tabs" or widget == "menu" \
# or widget == "menu_item" or widget == "toggle_button":
# if (random.random() < 0.5):
# pyautogui.click(x, y)
# else:
# pyautogui.rightClick(x, y)
# elif widget == "list" or widget == "scroll_bar" or widget == "slider":
# x = random.randint(x_start, x_end)
# y = random.randint(y_start, y_end)
# pyautogui.moveTo(x, y)
# action = random.random()
# if action < 0.4:
# pyautogui.click(x, y)
# elif action < 0.5:
# pyautogui.rightClick(x, y)
# else:
# pyautogui.click(x, y)
# x = random.randint(x_start, x_end)
# y = random.randint(y_start, y_end)
# pyautogui.dragTo(x, y)
# pyautogui.mouseUp(x, y)
# pyautogui.click(x, y)
# elif widget == "tree":
# pyautogui.click(x, y)
# x = random.randint(x_start, x_end)
# y = random.randint(y_start, y_end)
# pyautogui.moveTo(x, y)
#
# if (random.random() < 0.5):
# pyautogui.doubleClick(x, y)
# else:
# pyautogui.rightClick(x, y)
#
# x = random.randint(x_start, x_end)
# y = random.randint(y_start, y_end)
#
# pyautogui.moveTo(x, y)
#
# pyautogui.click(x, y)
# elif widget == "text_field":
# if (random.random() < 0.5):
# pyautogui.click(x, y)
# else:
# pyautogui.rightClick(x, y)
# pyautogui.typewrite(input_string, interval=0.01)
# # if random.random() < 0.2:
# # pyautogui.press('enter')
# elif widget == "combo_box":
#
# if (random.random() < 0.2): #press right button of cbox
# x = x_start + ((x_end-x_start)*0.85)
#
# if (random.random() < 0.5):
# pyautogui.click(x, y)
# else:
# pyautogui.rightClick(x, y)
#
# event = event + 1
# next_y = best_box[2]+random.random()*0.5
# x = int(max(app_x, min(app_x + app_w - 10, app_x + (best_box[1]*app_w))))
# y = int(max(app_y, min(app_y + app_h - 10, app_y + ((next_y)*app_h))))
#
# pyautogui.click(x, y)
#
# y_start = max(0, min(height, int(height*(next_y - best_box[4]/2))-10))
# y_end = max(0, min(height, int(height*(next_y+best_box[4]/2))+10))
#
# x_start = max(0, min(width, int(width*(best_box[1]-best_box[3]/2))-10))
# x_end = max(0, min(width, int(width*(best_box[1]+best_box[3]/2))+10))
# image_clicked = raw_image[y_start:y_end,
# x_start:x_end]
#
# output_img = output_dir + "/images/" + str(event)
#
# np.save(output_img, image_clicked)
#
# cv2.imwrite(output_img + ".jpg", image_clicked)
# else:
# print(widget, "unrecognised")
#
# pyautogui.mouseUp(x, y)
events = []
random_interaction = random.random()
if random_interaction < 0.888888888888: # just a normal click
if random.random() < 0.8:
events.append(MouseEvent(time.time(), 0, x, y))
pyautogui.click(x, y)
else:
events.append(MouseEvent(time.time(), 1, x, y))
else: # click and type 'Hello world!'
events.append(MouseEvent(time.time(), 0, x, y))
events.append(KeyEvent(time.time(), input_string))
for event in events:
event.perform()