def slider(window, target):
sleep(1)
mouse = Controller()
intervals = 10
short_delay = 0.1
if target['shape']['type'] == 'rect':
step = (target['shape']['width'] - target['shape']['x']) / intervals
position = (window['x'] + target['shape']['x'] + target['shape']['width'] / 2, \
window['y'] + target['shape']['y'] + target['shape']['height'] / 2)
elif target['shape']['type'] == 'circ':
step = (3 * target['shape']['radius'] -
target['shape']['centerX']) / intervals
position = (window['x'] + target['shape']['centerX'], \
window['y'] + target['shape']['centerY'])
elif target['shape']['type'] == 'poly':
step = 20
position = (window['x'] + target['shape']['centerX'], \
window['y'] + target['shape']['centerY'])
mouse.position = position
sleep(short_delay)
for i in range(intervals):
mouse.press(Button.left)
sleep(short_delay)
position = (position[0] + step, position[1])
mouse.position = position
sleep(short_delay)
mouse.release(Button.left)
sleep(1)
screenshot(window)
def draw(self):
img = self.get_color_pixels()
mouse = Controller()
# print(img.shape)
sleep(3)
x = self.startX
y = self.startY
mouse.position = (x, y)
for i in range(len(img)):
for j in range(len(img[i])):
if img[i][j] < 145:
mouse.press(Button.left)
if j < (len(img[i]) - 1):
if img[i][j + 1] < 145:
x += self.pwidth
mouse.position = (x, y)
continue
sleep(0.1)
mouse.release(Button.left)
x += self.pwidth
mouse.position = (x, y)
else:
x += self.pwidth
mouse.position = (x, y)
x = self.startX
y += self.pheight
mouse.position = (x, y)
def drag(window, target):
sleep(1)
mouse = Controller()
intervals = 10
short_delay = 0.1
step = 20 # (target['rect']['height'] - target['rect']['y']) / intervals
if target['shape']['type'] == 'rect':
position = (window['x'] + target['shape']['x'] + target['shape']['width'] / 2, \
window['y'] + target['shape']['y'] + 1)
elif target['shape']['type'] == 'circ':
position = (window['x'] + target['shape']['centerX'], \
window['y'] + target['shape']['centerY'])
elif target['shape']['type'] == 'poly':
position = (window['x'] + target['shape']['centerX'], \
window['y'] + target['shape']['centerY'])
orig_position = position
mouse.position = position
sleep(short_delay)
for i in range(intervals):
mouse.press(Button.left)
sleep(short_delay)
position = (position[0], position[1] + step)
mouse.position = position
sleep(short_delay)
mouse.release(Button.left)
sleep(2)
screenshot(window)
mouse.position = orig_position
sleep(short_delay)
def swipe(self, p1, p2, duration=0.5, steps=5, fingers=1, button='left'):
if button is "middle":
button = Button.middle
elif button is "right":
button = Button.right
elif button is "left":
button = Button.left
else:
raise ValueError("Unknow button: " + button)
x1, y1 = p1
x2, y2 = p2
x1 = x1 + self.monitor["left"]
x2 = x2 + self.monitor["left"]
y1 = y1 + self.monitor["top"]
y2 = y2 + self.monitor["top"]
ratio_x = self.monitor["width"] / self.singlemonitor["width"]
ratio_y = self.monitor["height"] / self.singlemonitor["height"]
x2 = x1 + (x2 - x1) / ratio_x
y2 = y1 + (y2 - y1) / ratio_y
m = Controller()
interval = float(duration) / (steps + 1)
m.position = (x1, y1)
m.press(button)
time.sleep(interval)
for i in range(1, steps + 1):
m.move(
int((x2 - x1) / steps),
int((y2 - y1) / steps)
)
time.sleep(interval)
m.position = (x2, y2)
time.sleep(interval)
m.release(button)
def mouse_(word='a'):
while 1:
if word == 'a':
word = speech_recognizer()
mouse = Controller()
if 'down' in word:
mouse.move(0, 40)
elif 'up' in word or 'upper side' in word:
mouse.move(0, -40)
elif 'right' in word:
mouse.move(40, 0)
elif 'left' in word:
mouse.move(-40, 0)
elif 'on first link' in word:
mouse.position = (250, 300)
elif 'first link' in word:
mouse.position = (250, 300)
mouse.press(Button.left)
mouse.release(Button.left)
elif 'at middle' in word:
mouse.position(670, 400)
if 'left click' in word or 'click left' in word:
mouse.press(Button.left)
mouse.release(Button.left)
if 'right click' in word or 'click right' in word:
mouse.press(Button.right)
mouse.release(Button.right)
elif 'stop song' in word or 'play' in word or 'paly song' in word:
mouse.position = (400, 300)
mouse.press(Button.left)
mouse.release(Button.left)
else:
return word
word = 'a'
def onClickRecordStart(self):
command = "pgrep -l %s | awk '{print $1}'" %(self.current_game)
child = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
result = child.communicate()[0].strip()
game_started = False
options = kCGWindowListOptionOnScreenOnly
geometry = None
while game_started == False:
window_list = CGWindowListCopyWindowInfo(options, kCGNullWindowID)
for window in window_list:
pid = window['kCGWindowOwnerPID']
if str(result) == str(pid):
geometry = window['kCGWindowBounds']
print geometry
game_started = True
applescript = os.getcwd()+'/AppleScripts/recorder.scpt'
command = 'osascript '+applescript
os.system(command)
mouse =Controller()
time.sleep(1)
mouse.position = (geometry['X'], geometry['Y']+(geometry['Height']-self.height))
time.sleep(1)
print 'start : ('+str(geometry['X'])+', '+str(geometry['Y']+(geometry['Height']-self.height))+')'
mouse.press(Button.left)
time.sleep(1)
mouse.position = (geometry['X']+geometry['Width'], geometry['Y']+geometry['Height'])
print 'start : ('+str(geometry['X']+geometry['Width'])+', '+str( geometry['Y']+geometry['Height'])+')'
time.sleep(1)
mouse.release(Button.left)
time.sleep(1)
mouse.position = ((2*geometry['X']+geometry['Width'])/2, (2*geometry['Y']+2*geometry['Height']-self.height)/2)
time.sleep(1)
mouse.click(Button.left, 1)
def serial_read():
ser = serial.Serial("COM11", 9600)
#Balance
xx = 682
dx = 245
yy = 550
mouse_factor = 5
mouse = Controller()
mouse.position = (xx, yy)
while True:
s1 = int.from_bytes(ser.read(), "big")
s2 = int.from_bytes(ser.read(), "big")
if (s1 == 255 & s2 == 255):
d1 = int.from_bytes(ser.read(), "big")
d2 = int.from_bytes(ser.read(), "big")
data = int.from_bytes(bytes([d1, d2]),
byteorder='big',
signed=True)
cksum = int.from_bytes(ser.read(), "big")
cksum0 = sum([s1, s2, d1, d2]) % 256
if (cksum == cksum0):
deltax = xx + (data / mouse_factor)
if (xx + dx > deltax > xx - dx):
mouse.position = (deltax, yy)
def click(x, y, delay, num=2):
mouse = Controller()
mouse.position = (x, y)
for i in range(1, num):
i = i
time.sleep(delay)
mouse.position = (x, y)
mouse.click(Button.left, 1)
def RandomMouse():
#
mouse = Controller()
#
mouse.position = (1500, 1500)
#
mouse.position = (random.randint(-1000, 1000), random.randint(-1000, 1000))
#
time.sleep(1)
def mouseFunc(event):
mouse = Controller()
app = wx.App(False)
(sx, sy) = wx.GetDisplaySize()
(camx, camy) = (320, 240)
lowerBound = np.array([33, 80, 40])
upperBound = np.array([102, 255, 255])
cam = cv2.VideoCapture(0)
kernelOpen = np.ones((5, 5))
kernelClose = np.ones((20, 20))
pinchFlag = 0
while True:
ret, img = cam.read()
img = cv2.resize(img, (340, 220))
imgHSV = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
mask = cv2.inRange(imgHSV, lowerBound, upperBound)
maskOpen = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernelOpen)
maskClose = cv2.morphologyEx(maskOpen, cv2.MORPH_CLOSE, kernelClose)
maskFinal = maskClose
conts, h = cv2.findContours(maskFinal.copy(), cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_NONE)
if (len(conts) == 2):
if (pinchFlag == 1):
pinchFlag = 0
mouse.release(Button.left)
x1, y1, w1, h1 = cv2.boundingRect(conts[0])
x2, y2, w2, h2 = cv2.boundingRect(conts[1])
cv2.rectangle(img, (x1, y1), (x1 + w1, y1 + h1), (255, 0, 0), 2)
cv2.rectangle(img, (x2, y2), (x2 + w2, y2 + h2), (255, 0, 0), 2)
cx1 = x1 + w1 // 2
cy1 = y1 + h1 // 2
cx2 = x2 + w2 // 2
cy2 = y2 + h2 // 2
cx = (cx1 + cx2) // 2
cy = (cy1 + cy2) // 2
cv2.line(img, (cx1, cy1), (cx2, cy2), (255, 0, 0), 2)
cv2.circle(img, (cx, cy), 2, (0, 0, 255), 2)
mouseLoc = (sx - (cx * sx // camx), cy * sy // camy)
mouse.position = mouseLoc
while mouse.position != mouseLoc:
pass
elif (len(conts) == 1):
x, y, w, h = cv2.boundingRect(conts[0])
if (pinchFlag == 0):
pinchFlag = 1
#mouse.press(Button.left)
cv2.rectangle(img, (x, y), (x + w, y + h), (255, 0, 0), 2)
cx = x + w // 2
cy = y + h // 2
cv2.circle(img, (cx, cy), (w + h) // 4, (0, 0, 255), 2)
mouseLoc = (int(sx - (cx * sx // camx)), int(cy * sy // camy))
mouse.position = mouseLoc
while mouse.position != mouseLoc:
pass
cv2.imshow("cam", img)
cv2.waitKey(5)
def focus_windows(): # Only way to focus UI on Windows
mouse = Controller()
# mouse position before focus
mouse_pos_before = mouse.position
# changing the mouse position for click
target_pos_x = ui.pos().x() + ui.textbox.pos().x()
target_pos_y = ui.pos().y() + ui.textbox.pos().y()
mouse.position = (target_pos_x, target_pos_y)
mouse.click(Button.left)
mouse.position = mouse_pos_before
def focus_ui(): # Only way I could think of to properly focus the ui
mouse = Controller()
# mouse position before focus
mouse_pos_before = mouse.position
# changing the mouse position for click
target_pos_x = ui.pos().x() + ui.textbox.pos().x()
target_pos_y = ui.pos().y() + ui.textbox.pos().y()
mouse.position = (target_pos_x, target_pos_y)
mouse.click(Button.left)
mouse.position = mouse_pos_before
def DoSomething(self):
# self.TimerSleep()
mouse = Controller()
while 1:
mouse.position = (152, 600)
mouse.click(Button.left, 1)
time.sleep(5)
mouse.position = (152, 660)
mouse.click(Button.left, 1)
time.sleep(10)
def mouse_drag(start, finish):
""" Fonction qui va effectuer un déplacement depuis start jusqu'à finish"""
from pynput.mouse import Button, Controller
mouse = Controller()
mouse.position = start
mouse.press(Button.left)
mouse.position = finish
time.sleep(0.1)
mouse.position = (finish[0] + 1, finish[1])
time.sleep(0.1)
mouse.release(Button.left)
def move_puzzle(from_, to_, array):
print(from_, to_)
temp = array[from_]
array[from_] = array[to_]
array[to_] = temp
mouse = Controller()
mouse.position = positions[from_]
mouse.press(Button.left)
mouse.position = positions[to_]
mouse.release(Button.left)
def replayEdit(file_name):
global APP
screen_width = APP.winfo_screenwidth()
screen_height = APP.winfo_screenheight()
different_screen = False
instruction_list = []
df = pandas.read_csv(file_name)
df = df[df['Mouse Drag'] != 'place']
df_screen_width = df['X_loc'].iloc[0]
df_screen_height = df['Y_loc'].iloc[0]
df_screen_width = typeToInt(df_screen_width)
df_screen_height = typeToInt(df_screen_height)
width_ratio = float(screen_width) / float(df_screen_width)
height_ratio = float(screen_height) / float(df_screen_height)
if width_ratio != 1.0 or height_ratio != 1.0:
different_screen = True
print('Calculating different screen dimensions')
instruction_list = df['INSTRUCTION'].to_list()
mouse = Controller()
keyboard = KeyboardController()
for row in instruction_list[2:]:
row = literal_eval(row)
print(row)
try:
if different_screen == True:
mouse.position = int(round(width_ratio * float(row[0]))), int(
round(height_ratio * float(row[1])))
else:
mouse.position = int(row[0]), int(row[1])
except:
pass
if row[3] == 'True':
try:
print(eval(row[2]))
mouse.press(eval(row[2]))
except:
print(row[2])
keyboard.press(checkBut(row[2]))
if row[3] == 'False':
try:
print(eval(row[2]))
mouse.release(eval(row[2]))
except:
print(row[2])
keyboard.release(checkBut(row[2]))
try:
dy = int(row[3])
mouse.scroll(
0, dy
) ##the scrolling part could be tested in an automated script. Get the scroll right until on a wikipedia page you get the same word copied and pasted twice
except:
pass
time.sleep(row[-1])
def processing(operator):
global FlowerpotCoords
global ButtonCoords
mouse = Controller()
for fc, bc in zip(FlowerpotCoords, ButtonCoords):
var.set(operator + 'ing......')
mouse.position = fc
mouse.click(Button.right, 1)
time.sleep(0.5)
mouse.position = bc
mouse.click(Button.left, 1)
time.sleep(0.5)
def click(threadName, delay, x, y, num):
mouse = Controller()
mouse.position = (x, y)
print('The current pointer position is {0} and time is {1}'.format(
mouse.position, time.ctime(time.time())))
for i in range(1, num):
if exitFlag:
break
time.sleep(delay)
# print ("%s: %s" % (threadName, time.ctime(time.time())))
mouse.position = (x, y)
mouse.click(Button.left, 1)
def replayEdit(file_name):
global APP
screen_width = APP.winfo_screenwidth()
screen_height = APP.winfo_screenheight()
different_screen = False
instruction_list = []
df = pandas.read_csv(file_name)
df = df[df['Mouse Drag'] != 'place'] #this is the key line here. We're essentially eliminating all non eventful rows. The idea is to automatically speed up your recordings
df_screen_width = df['X_loc'].iloc[0] #while this is a simple way of doing that, refer to the askForScreenshot() and getCorners() functions for a more complex speedup
df_screen_height = df['Y_loc'].iloc[0]
df_screen_width = typeToInt(df_screen_width)
df_screen_height = typeToInt(df_screen_height)
width_ratio = float(screen_width)/float(df_screen_width)
height_ratio = float(screen_height)/float(df_screen_height)
if width_ratio != 1.0 or height_ratio != 1.0:
different_screen = True
print('Calculating different screen dimensions')
instruction_list = df['INSTRUCTION'].to_list()
mouse = Controller()
keyboard = KeyboardController()
for row in instruction_list[2:]:
row = literal_eval(row)
print(row)
try:
if different_screen == True: #refer to replay and safeReplay() functions
mouse.position = int(round(width_ratio * float(row[0]))),int(round(height_ratio * float(row[1]))) #this way is very slow if you have a different screen
else:
mouse.position = int(row[0]),int(row[1])
except:
pass
if row[3] == 'True': #this will be True on any press
try:
print(eval(row[2]))
mouse.press(eval(row[2])) #so we try mouse, for the button in row[2]
except:
print(row[2])
keyboard.press(checkBut(row[2])) #if not mouse it must have been a keyboard press
if row[3] == 'False': #row[3] will be 'False' on any release
try:
print(eval(row[2]))
mouse.release(eval(row[2])) #so first, we try the button on mouse. Eval is essential here because we need a class object, not a string as the parameter
except:
print(row[2])
keyboard.release(checkBut(row[2])) #if the mouse release doesn't work, it must be a button release. This actually shouldn't work but it does because when you save to a csv
#your pynput button objects get converted to strings
try:
dy = int(row[3])
mouse.scroll(0,dy) #scrolling is pretty broken. It'll work if you scroll all the way to a buffer or page limit. But it's not accurate enough to scroll wikipedia and copy and paste words
except: #I used try and excepts because they're fast and readable
pass
time.sleep(row[-1]) #pages load at different rates, apps too. This sleeps for as long as you did during recording
def mouse_click(*args, position = None):
self = args[0]
mouse = Controller()
if position == None:
position = Controller().position
mouse.position = position
mouse.press(Button.left)
mouse.release(Button.left)
if self.name == 'plot_app':
mouse.position = self.parent_app.center_position
mouse.press(Button.left)
mouse.release(Button.left)
def calibrate():
print("Calibrate Start")
mouse = MController()
h = root.winfo_screenheight()
for i in range(5):
mouse.position = mouse.position[0], h
time.sleep(0.05)
mouse.position = mouse.position[0], h
time.sleep(0.05)
for i in range(180):
Controller.move_by_one_deg(1)
print(i)
print("Calibrate End")
def voltar(
): #Função minimiza o player e fecha ele basta tirar um print da tela, abrir no paint e substituir os valores.
mouse = Controller()
mouse.position = (1580, 1184)
time.sleep(0.5)
mouse.click(Button.left, 1)
print('Reprodutor minimizado.')
time.sleep(3.0)
mouse.position = (1347, 153)
mouse.click(Button.left, 1)
time.sleep(3.0)
screen_shot()
print('Reprodutor fechado')
screen_shot()
def reproduzir(): # Coloca o player em tela cheia e reproduz
mouse = Controller()
mouse.position = (
1327, 784
) # Localização do botão de tela cheia na minha tela, basta tirar um print da sua tela quando o player estiver aberto e abrir ele no paint para ver a localização do item na sua tela.
mouse.click(Button.left, 1)
time.sleep(1.0)
print('Reprodutor colocado em tela cheia.')
time.sleep(2.0)
mouse.position = (
20, 1186
) # Localização do potão player na minha tela, basta sustituir pela posição da sua tela.
mouse.click(Button.left, 1)
print('Video reproduzindo.')
mouse.position = (1, 1)
def main():
mouse = Controller()
# Read pointer position
print("The current pointer position is {0}".format(mouse.position))
print("Now please stare at your mouse, not the terminal :)")
print("The mouse will move to the center of the screen\n")
time.sleep(MOUSE_SLEEP_INTERVAL)
# Set pointer position
mouse.position = (1400, 900)
print("Now we have moved it to {0}".format(mouse.position))
print("A move based on absolute position")
print("Then the mouse will move to the right\n")
time.sleep(MOUSE_SLEEP_INTERVAL)
# Move pointer relative to current position
mouse.move(50, 0)
print("Now we have moved it to {0}".format(mouse.position))
print("A move based on relative position")
print("Then we will click the right mouse button\n")
time.sleep(MOUSE_SLEEP_INTERVAL)
# Press and release
mouse.press(Button.right)
mouse.release(Button.right)
print(
"Then we will move to the left and double click the left mouse button\n"
)
time.sleep(MOUSE_SLEEP_INTERVAL)
mouse.move(-50, 0)
# Double click; this is different from pressing and releasing
mouse.click(Button.left, 2)
def process(self):
mouse = Controller()
mouse.position = (self.position.x, self.position.y)
if self.pressed:
mouse.press(self.button)
else:
mouse.release(self.button)
def mouse_move_worker(objectX, objectY, mouse_state):
""" Separate process worker to move mouse and perform clicks"""
import tkinter
from pynput.mouse import Button, Controller
import time
mouse = Controller()
x, y = tkinter.Tk().winfo_screenwidth(), tkinter.Tk().winfo_screenheight()
while True:
if mouse_state.value == MOUSE_CLICK:
time.sleep(0.2)
mouse.press(Button.left)
mouse_state.value = MOUSE_RELEASE
if mouse_state.value == MOUSE_DRAG:
time.sleep(0.2)
mouse.press(Button.left)
mouse_state.value = MOUSE_DRAGGING
if (mouse_state.value == MOUSE_RELEASE):
mouse.release(Button.left)
mouse_state.value = MOUSE_NULL
if (objectX.value > 0 and objectY.value > 0):
mouse.position = (int(
(1 - objectX.value) * x), int(objectY.value * y))
def prayClicker():
time.sleep(12)
button = Button.left
mouse = Controller()
while (True):
# Laptop Coords
xcoord = random.randint(1705, 1720)
ycoord = random.randint(110, 126)
mouse.position = (xcoord, ycoord)
mouse.click(button)
time.sleep(0.4)
mouse.click(button)
sleeptime = random.randint(31, 54) # 110 - 250
# Alert for upcoming click
print("P: Going to sleep for: " + str(sleeptime) + " seconds.")
for k in range(sleeptime, 0, -1):
if (k == 30):
print("P: 30s left.")
elif (k == 20):
print("P: 20s left.")
elif (k == 10):
print("P: 10s left.")
time.sleep(1)
# Random wait that isnt a whole integer
time.sleep(random.random())
def repeat_one_time():
from pynput.mouse import Button
from pynput.mouse import Controller as Mouse_Controller
from pynput.keyboard import Controller as Keyboard_Controller
mouse = Mouse_Controller()
keyboard = Keyboard_Controller()
record_data = pickle.load(open("record.p", "rb"))
print(record_data)
for action in record_data:
if action['type'] == 'mouse':
mouse.position = (int(action['x']), int(action['y']))
if action['action'] == 'press':
mouse.press(action['button'])
elif action['action'] == 'release':
mouse.release(action['button'])
elif action['type'] == 'keyboard':
if action['action'] == 'release':
keyboard.release(action['key'])
elif action['action'] == 'press':
keyboard.press(action['key'])
time.sleep(0.1)
def browser(word):
mouse = Controller()
if 'next recomended song' in word:
mouse.position = (1000, 200)
mouse.press(Button.left)
mouse.release(Button.left)
print('Jarvis :next song played')
say('next song played')
log_('next song played')
else:
try:
print('Jarvis :searchng in progress...please wait!')
say('searchng in progress...please wait!')
log_('searchng in progress...please wait!')
#word=speech_recogniser()
for link in search(word, tld='co.in', num=1, stop=1, pause=2):
print("Jarvis :obtaied link :" + link)
log_(link)
wb.open(link)
if 'youtube' in word:
t.sleep(1)
mouse = Controller()
#mouse.position=(400,300)
#mouse.press(Button.left)
#mouse.release(Button.left)
print('jarvis :done')
print("Jarvis :" + 'searching done!')
say('searching done!')
log_('searching done!')
except:
p = 'sorry!\n word not recognized try again.'
print(p)
log_(p)
return browser()
def get_names(self):
mouse = Controller()
mouse.position = (
674, 827
) #YOU WILL NEED TO CALIBRATE THIS DEPENDING OF YOUR MONITOR, THE CURSOR MUST GO TO THE PAGE DOWN ARROW
mouse.press(Button.left)
time.sleep(90)
mouse.release(Button.left)
scroll_box = self.driver.find_element_by_xpath("/html/body/div[4]/div")
last_ht, ht = 0, 1
while last_ht != ht:
last_ht = ht
time.sleep(1)
ht = self.driver.execute_script(
"""
arguments[0].scrollTo(0, arguments[0].scrollHeight);
return arguments[0].scrollHeight;
""", scroll_box)
links = scroll_box.find_elements_by_tag_name('a')
names = [name.text for name in links if name.text != '']
# close button
time.sleep(3)
self.driver.find_element_by_xpath(
'/html/body/div[4]/div/div/div[1]/div/div[2]/button').click()
self.whitelist(names)
#-*-coding:utf-8-*-
from pynput.mouse import Button, Controller
import time
import win32clipboard as w
import win32con
from controlKeyboard import doKeyboard,altUp,down,enter,LaoGongTV,ctrlTab,ctrlW,space,ctrlV,keyFill,ctrlShiftI
mouse = Controller()
def setText(aString):
w.OpenClipboard()
w.EmptyClipboard()
w.SetClipboardData(win32con.CF_UNICODETEXT, aString)
w.CloseClipboard()
for i in range(100):
mouse.position = (1702,1006)
mouse.click(Button.left,1)
time.sleep(0.5)
codes = '''
document.getElementsByClassName("p-txt")[%s].innerText;
''' % str(i)
setText(codes)
ctrlV()
# time.sleep(2)
enter()
