def exeMaximize(tokens, mode):
if tokens:
gui.showError("Incorrect\nusage of maximize")
handle = win32gui.GetForegroundWindow()
win32gui.ShowWindow(handle, win32con.SW_MAXIMIZE)
def exeKeystroke(tokens, mode):
if len(tokens) == 1:
try:
keyboard.press_and_release(tokens[0])
except ValueError as e:
log.error(str(e))
log.error("bad keystroke '{}'".format(tokens[0]))
gui.showError('Invalid usage')
else:
log.Logger.log(log.ParseError.TYPE, tokens)
def exeFind(tokens):
"""Finds text on the page."""
if len(tokens) > 0 and tokens[0] == "PHRASE":
send_message(encode_message([KeyboardMessage('/')]))
time.sleep(0.5)
keyboard.write(' '.join(tokens[1:]))
time.sleep(0.25)
keyboard.press_and_release('enter')
else:
gui.showError("Unrecognized\nFind sequence")
time.sleep(1)
def exeTerminate(tokens, mode):
if currentApp() == 'Firefox':
gui.showError('Closing Firefox\nwould close SpeechV')
return ([], mode)
handles = window_properties.getMainWindowHandles(
processNameOf(currentApp()))
#try:
# target = handles[0] # choose arbitrary window to terminate if more than one
#except IndexError:
# log.error("No window to terminate for current app '{}'".format(currentApp()))
# return ([], mode)
if not handles:
log.error("No window to terminate for current app '{}'".format(currentApp()))
return ([], mode)
for handle in handles:
win32api.SendMessage(handle, win32con.WM_DESTROY, None, None)
def exeLaunch(tokens, mode):
"""Launch application, or focus it if the app is already launched"""
if len(tokens) != 1:
gui.showError("Unrecognized\nApplication")
log.warn("unrecognized application for launch cmd: '{}'".format(' '.join(tokens)))
elif tokens[0] == 'FIREFOX':
# technically, this will always be open if speechv is running. Focus
# the application instead
exeFocus(['FIREFOX'], mode)
elif tokens[0] == 'WORD':
# check if it's already open
handles = window_properties.getMainWindowHandles(processNameOf('WORD'))
if handles:
exeFocus(['WORD'], mode)
else:
# launch it. To complete this project in a reasonable amount of time
# we hardcode it. Windows has spotty support for this type of stuff
subprocess.Popen([r'C:\Program Files (x86)\Microsoft Office\root\Office16\WINWORD.EXE'])
def exeFocus(tokens, mode):
# https://stackoverflow.com/questions/44735798/pywin32-how-to-get-window-handle-from-process-handle-and-vice-versa
log.debug("app to focus: '{}'".format(tokens[0]))
if len(tokens) != 1:
gui.showError("Incorrect\nusage of focus")
log.warn("focus used without exactly one token")
return ([], mode)
processName = processNameOf(tokens[0])
log.debug("processName: '{}'".format(processName))
handles = window_properties.getMainWindowHandles(processName, expect_one=True)
if not handles:
gui.showError("No app to focus")
return ([], mode)
# NOTE: we choose an arbitrary handle if there's more than one
handle = handles[0]
try:
win32gui.SetForegroundWindow(handle)
except Exception as e:
log.error(str(e))
try:
# SetForegroundWindow is unreliable
# workarounds: https://stackoverflow.com/questions/3772233/
win32gui.ShowWindow(handle, win32con.SW_MINIMIZE)
win32gui.ShowWindow(handle, win32con.SW_SHOWNORMAL)
win32gui.SetForegroundWindow(handle)
except Exception as e:
log.error(str(e))
log.error("couldn't focus app '{}'".format(tokens[0]))
gui.showError("Couldn't focus app")
return ([], mode)
# De-minimize window if necessary
if win32gui.IsIconic(handle):
win32gui.ShowWindow(handle, win32con.SW_SHOWNORMAL)
def parseImpl(self, tokens, levelDict = None):
log.info("current mode: ", self.mode.name)
if not tokens:
return
# TODO: better method of accepting letters to follow
# currently just takes all tokens as individual letters and sends
# them on assuming that there's no point in issuing commands
# before the link is followed. Also, there should not be more than 3
if self.mode == GlobalMode.FOLLOW and currentApp() == 'Firefox':
# handle switch specially so that alt-tabbing works with the prompt
if tokens[0] == 'SWITCH' and len(tokens) == 1:
commands.exeSwitch(tokens[1:], self.mode)
return
elif tokens[0] == 'CANCEL' and len(tokens) == 1:
_, mode = commands.exeCancel([], self.mode)
self.mode = mode
return
if all(len(tok) == 1 for tok in tokens):
# Take a page out of WordForwarder and send it as long as we're
# still following and it's a letter. Trust that the user won't
# say arbitrary letters outside of this.
enumerated_keys = [commands.KeyboardMessage(tok.lower()) for tok in tokens]
send_message(encode_message(enumerated_keys))
else:
# Some other command so go back to navigate mode and retry parsing
_, mode = commands.exeCancel([], self.mode)
self.mode = mode
self.parseImpl(tokens, levelDict)
return
# See the comment by self.commands in __init__ for the details
def handleCmdRet(ret):
if ret is not None:
leftoverTokens, self.mode = ret[0], ret[1]
self.parseImpl(leftoverTokens, self.mode)
if self.mode == GlobalMode.FOLLOW and currentApp() == 'Microsoft Word':
if tokens[0] == "INSERT":
self.mode = GlobalMode.INSERT
return
if tokens[0] == 'SWITCH' and len(tokens) == 1:
commands.exeSwitch(tokens[1:], self.mode)
return
# forward tokens to wordForwarder
ret = self.wordForwarder.followWord(tokens)
handleCmdRet(ret)
return
if self.mode == GlobalMode.NAVIGATE and len(tokens) == 1:
if tokens[0] == "INSERT":
self.mode = GlobalMode.INSERT
# FIXME: we currently don't use this feature. Maybe we want to remove
# it so that the code is easier to read?
if levelDict is None:
levelDict = self.commands
w, rest = tokens[0], tokens[1:]
if w in levelDict:
if isinstance(levelDict[w], dict):
self.parseImpl(rest, levelDict[w])
else:
ret = levelDict[w](rest, self.mode)
handleCmdRet(ret)
else:
log.info('forwarding to current app: ', currentApp())
if currentApp() == 'Firefox':
ret = commands.forwardBrowser(tokens, self.mode)
handleCmdRet(ret)
elif currentApp() == 'Microsoft Word':
ret = self.wordForwarder.forward(tokens, self.mode)
handleCmdRet(ret)
elif ' '.join(tokens) in self.config['MACROS']:
self.exeMacro((self.config['MACROS'][' '.join(tokens)]))
else:
gui.showError("Unrecognized\nCommand")
log.warn("Command not found")
raise ValueError("Unrecognized Command")
def parse(self, command):
self.ready = False
log.debug("parsing command: '{}'".format(command))
command = command.strip().upper()
if command == 'WAKE':
if self.mode == GlobalMode.SLEEPING:
self.mode = GlobalMode.NAVIGATE
return
else:
gui.showError("Cannot wake\n while awake")
log.warn("wake command attempted while not sleeping")
return
elif self.mode == GlobalMode.SLEEPING:
return # do not execute commands while sleeping
isFullyHandled = self.macroManager.interceptCommand(command)
if isFullyHandled:
return
if len(command) == 1:
command = command.lower()
def handleCmdRet(ret):
if ret is not None:
leftoverTokens, self.mode = ret[0], ret[1]
if self.mode == GlobalMode.NAVIGATE or self.mode == GlobalMode.FOLLOW:
command = re.sub('[!@#$\']', '', command)
text = re.findall(r"[a-zA-Z]+", command)
log.info("Tokens parsed: {}".format(text))
self.parseImpl(text)
elif self.mode == GlobalMode.INSERT:
if command == 'NAVIGATE':
self.mode = GlobalMode.NAVIGATE
elif command == 'HIGHLIGHT' and currentApp() == 'Microsoft Word':
self.mode = GlobalMode.NAVIGATE
#Send as a list or it will end up as "H I G H L I G H T"
self.wordForwarder.forward(["HIGHLIGHT"], self.mode)
# #if currentApp() == 'Microsoft Word':
# # self.wordmode = WordMode.NAVIGATE
# elif command == 'HIGHLIGHT' and currentApp() == 'Microsoft Word':
# self.mode = GlobalMode.NAVIGATE
# #self.wordmode = WordMode.HIGHLIGHT
elif command == 'ENTER':
keyboard.press_and_release('enter')
elif command == 'NEW PARAGRAPH':
keyboard.press_and_release('enter')
keyboard.press_and_release('enter')
else:
#In this case, send the top application the text to type it
command = re.sub('[!@#$\']', '', command)
text = re.findall(r"[a-zA-Z]+", command)
text = [word.lower() for word in text]
#If we ended the last command with a period
if self.newSentence:
text[0] = text[0].title()
self.newSentence = False
if text[-1].upper() == "PERIOD":
#Remove the word period and replace with a dot
#Either as its own string or attached to the last word
text = text[:-1]
if len(text) == 0:
text[0] = '.'
else:
text[-1] += '.'
self.newSentence = True
elif text[-1].upper() == "COMMA":
#Remove the word comma and replace with ','
#Either as its own string or attached to the last word
text = text[:-1]
if len(text) == 0:
text[0] = ','
else:
text[-1] += ','
command = ' '.join(text)
log.info("Sending: \"{}\" to top application".format(command))
keyboard.write(command + ' ')
#keys = [commands.KeyboardMessage(ch) for ch in command]
#send_message(encode_message(keys))
elif self.mode == GlobalMode.SETTINGS:
command = re.sub('[!@#$\']', '', command)
text = re.findall(r"[a-zA-Z]+", command)
ret = commands.forwardSettings(text)
handleCmdRet(ret)
# elif self.mode == GlobalMode.HELP:
# command = re.sub('[!@#$\']', '', command)
# text = re.findall(r"[a-zA-Z]+", command)
# ret = commands.forwardHelp(text)
# handleCmdRet(ret)
else:
# Oh no! We have a bad mode. Hopefully going back to NAVIGATE saves us
log.error('unknown mode:', self.mode)
self.mode = GlobalMode.NAVIGATE
# the command was succesfull commit it if we need to
self.macroManager.conditionalCommit(command)
# sleep after parsing to allow commands to send appropriately
time.sleep(0.5)
def voiceLoop():
global restartLoop
config = settings.loadConfig()
AUDIO_TIMEOUT = config["SETTINGS"]["TIMEOUT"] # length of pause marking end of command
with open('command_set.txt', 'r') as myfile:
str_command_set = myfile.read()
command_set = str_command_set.split('\n')
in_debug_mode = False
if os.path.exists('DEBUG_FLAG'):
in_debug_mode = True
log.info("debug mode activated")
opened = False
while not opened:
try:
pipe = win32file.CreateFile(
r'\\.\pipe\named_pipe',
win32file.GENERIC_READ | win32file.GENERIC_WRITE,
win32file.FILE_SHARE_WRITE | win32file.FILE_SHARE_READ,
None, win32file.OPEN_EXISTING, 0, None)
opened = True
except Exception as e:
log.error("HELLO WORLD")
log.error(str(e))
log.error(traceback.format_exc())
time.sleep(1)
time.sleep(1)
else:
log.info("voice mode activated")
p = parsing.Parser()
with mic as source:
#Automatically adjust for ambient noise instead of calling calibrate
r.dynamic_energy_threshold = True
r.pause_threshold = AUDIO_TIMEOUT
while True:
try:
gui.ready()
raw_command = ''
if in_debug_mode:
message = win32file.ReadFile(pipe, 4096)
log.debug('pipe message: ', message[1].decode())
raw_command = message[1].decode()
else:
audio = r.listen(source)
# recognize speech using Google Cloud Speech API
log.debug("Pre recognize")
raw_command = recognize(audio, command_set)
gui.processing()
cmdPromptHandle = None
if in_debug_mode and not os.path.exists('BATCH_FLAG'):
keyboard.press_and_release("alt+tab")
time.sleep(1) # give OS time to alt-tab
if raw_command == -1:
raise ValueError("Failed to recognize speech")
else:
p.parse(raw_command)
if os.path.exists('BATCH_FLAG'):
# send an ACK to tell them we're ready for more input
win32file.WriteFile(pipe, 'ACK'.encode())
elif in_debug_mode:
time.sleep(1) # give the user time to see the result
commands.exeFocus(['CMD'], GlobalMode.NAVIGATE)
gui.updateCommands(raw_command)
except Exception as e:
log.error(str(e))
log.error(traceback.format_exc())
errorMsg = None
try:
log.debug('type(raw_command) = {}'.format(type(raw_command)))
# this looks incredibly dumb, but there's a reason: we really
# just want to check if raw_command is an integer. This could be
# refactored but #timepressure
if raw_command == -1:
errorMsg = '(google api failed!)'
else:
errorMsg = '(google api failed!)'
except ValueError:
errorMsg = 'error: ' + str(raw_command)
gui.updateCommands(errorMsg)
gui.showError("Error parsing\nTry again.")
if p.mode == GlobalMode.FOLLOW:
continue
gui.setMode(p.mode)
