def recognize(spec, choices_values, timeout): global RESULT RESULT = None grammar = Grammar("grammar") extras = [] for name, choices in choices_values.iteritems(): extras.append(Choice(name, dict((c,c) for c in choices))) Rule = type("Rule", (GrammarRule,),{"spec": spec, "extras": extras}) grammar.add_rule(Rule()) grammar.load() future = time.time() + timeout while time.time() < future: if RESULT is not None: break pythoncom.PumpWaitingMessages() time.sleep(.1) grammar.unload() print "RESULT:", RESULT return RESULT
def test_list_grammars(self): """ Verify that the 'list_grammars' RPC method works correctly. """ # Load a Grammar with three rules and check that the RPC returns the # correct data for them. g = Grammar("list_grammars_test") g.add_rule(CompoundRule(name="compound", spec="testing", exported=True)) g.add_rule(MappingRule(name="mapping", mapping={ "command a": ActionBase(), "command b": ActionBase() })) g.add_rule(Rule(name="base", element=Literal("hello world"), exported=False)) g.load() response = self.send_request("list_grammars", []) expected_grammar_data = { "name": g.name, "enabled": True, "active": True, "rules": [ {"name": "compound", "specs": ["testing"], "exported": True, "active": True}, {"name": "mapping", "specs": ["command a", "command b"], "exported": True, "active": True}, {"name": "base", "specs": ["hello world"], "exported": False, "active": True} ] } # Check that the loaded grammar appears in the result. It might not # be the only grammar and that is acceptable because dragonfly's # tests can be run while user grammars are loaded. try: self.assertIn("result", response) self.assertIn(expected_grammar_data, response["result"]) finally: g.unload()
class SwitchMode: def __init__(self, modeSwitcher): self.mode = "regular" if (SPEECHREC_ENABLED == True): # Create a grammar which contains and loads the command rule. self.grammar = Grammar( "Switch grammar" ) # Create a grammar to contain the command rule. twitchRule = TwitchModeRule() twitchRule.setModeSwitch(modeSwitcher) self.grammar.add_rule( twitchRule) # Add the command rule to the grammar. youtubeRule = YoutubeModeRule() youtubeRule.setModeSwitch(modeSwitcher) self.grammar.add_rule( youtubeRule) # Add the command rule to the grammar. browseRule = BrowseModeRule() browseRule.setModeSwitch(modeSwitcher) self.grammar.add_rule( browseRule) # Add the command rule to the grammar. heroesRule = HeroesModeRule() heroesRule.setModeSwitch(modeSwitcher) self.grammar.add_rule( heroesRule) # Add the command rule to the grammar. testingRule = TestingModeRule() testingRule.setModeSwitch(modeSwitcher) self.grammar.add_rule( testingRule) # Add the command rule to the grammar. excelLogModeRule = ExcelLogModeRule() excelLogModeRule.setModeSwitch(modeSwitcher) self.grammar.add_rule( excelLogModeRule) # Add the command rule to the grammar. excelModeRule = ExcelModeRule() excelModeRule.setModeSwitch(modeSwitcher) self.grammar.add_rule( excelModeRule) # Add the command rule to the grammar. def start(self): self.grammar.load() mute_sound() toggle_speechrec() def handle_input(self, dataDicts): if (SPEECHREC_ENABLED == True): pythoncom.PumpWaitingMessages() sleep(.1) else: print( "NO SPEECH RECOGNITION ENABLED - CANNOT SWITCH BETWEEN MODES") sleep(.5) def exit(self): self.grammar.unload() toggle_speechrec() turn_on_sound()
def test_list_grammars(self): """ Verify that the 'list_grammars' RPC method works correctly. """ # Load a Grammar with three rules and check that the RPC returns the # correct data for them. g = Grammar("list_grammars_test") g.add_rule(CompoundRule(name="compound", spec="testing", exported=True)) g.add_rule( MappingRule(name="mapping", mapping={ "command a": ActionBase(), "command b": ActionBase() })) g.add_rule( Rule(name="base", element=Literal("hello world"), exported=False)) g.load() response = self.send_request("list_grammars", []) expected_grammar_data = { "name": g.name, "enabled": True, "active": True, "rules": [{ "name": "compound", "specs": ["testing"], "exported": True, "active": True }, { "name": "mapping", "specs": ["command a", "command b"], "exported": True, "active": True }, { "name": "base", "specs": ["hello world"], "exported": False, "active": True }] } # Check that the loaded grammar appears in the result. It might not # be the only grammar and that is acceptable because dragonfly's # tests can be run while user grammars are loaded. try: self.assertIn("result", response) self.assertIn(expected_grammar_data, response["result"]) finally: g.unload()
def test_mimic(self): """ Verify that the 'mimic' RPC method works correctly. """ g = Grammar("mimic_test") g.add_rule(CompoundRule(name="compound", spec="testing mimicry", exported=True)) g.load() # Set the grammar as exclusive. # The sapi5shared engine apparently requires this for mimic() to # work, making the method kind of useless. This does not apply to # sapi5inproc. g.set_exclusiveness(True) response = self.send_request("mimic", ["testing mimicry"]) try: self.assertIn("result", response) self.assertEqual(response["result"], True) finally: g.set_exclusiveness(False) g.unload()
def test_mimic(self): """ Verify that the 'mimic' RPC method works correctly. """ g = Grammar("mimic_test") g.add_rule( CompoundRule(name="compound", spec="testing mimicry", exported=True)) g.load() # Set the grammar as exclusive. # The sapi5shared engine apparently requires this for mimic() to # work, making the method kind of useless. This does not apply to # sapi5inproc. g.set_exclusiveness(True) response = self.send_request("mimic", ["testing mimicry"]) try: self.assertIn("result", response) self.assertEqual(response["result"], True) finally: g.set_exclusiveness(False) g.unload()
def test_recognition_history_methods(self): """ Verify that the recognition history RPC methods work correctly. """ # Load a grammar for testing that recognitions are saved. g = Grammar("history_test") g.add_rule( CompoundRule(name="compound", spec="test history", exported=True)) g.load() g.set_exclusiveness(True) try: # Test 'register_history()'. response = self.send_request("register_history", []) self.assertIn("result", response) # Test that the method raises an error if used when the observer # is already registered. self.assertRaises(RuntimeError, self.send_request, "register_history", []) # Send a mimic and check that it is returned by the # 'get_recognition_history()' method. self.send_request("mimic", ["test history"]) response = self.send_request("get_recognition_history", []) self.assertIn("result", response) self.assertListEqual(response["result"], [["test", "history"]]) # Test 'unregister_observer()'. response = self.send_request("unregister_history", []) self.assertIn("result", response) # Test that the method raises an error if used when the observer # is not registered. self.assertRaises(RuntimeError, self.send_request, "unregister_history", []) finally: g.set_exclusiveness(False) g.unload()
class GrammarController(object): """Wraps grammars so they can be turned on and off by command.""" def __init__(self, name, grammars): self._controlled_grammars = grammars self.enabled = True rule = create_rule(name + "_mode", { name + " (off|close)": Function(lambda: self.disable()), name + " (on|open)": Function(lambda: self.enable()), }, exported=True) self._command_grammar = Grammar(name + "_mode") self._command_grammar.add_rule(rule) def enable(self): if not self.enabled: for grammar in self._controlled_grammars: grammar.enable() self.enabled = True def disable(self): if self.enabled: for grammar in self._controlled_grammars: grammar.disable() self.enabled = False def load(self): for grammar in self._controlled_grammars: grammar.load() self._command_grammar.load() def unload(self): for grammar in self._controlled_grammars: grammar.unload() self._command_grammar.unload()
class BaseMode: quadrant3x3 = 0 quadrant4x3 = 0 speech_commands = {} patterns = {} toggles = {} use_direct_keys = False input_release_lag = 0 def __init__(self, modeSwitcher, is_testing=False, repeat_delay=REPEAT_DELAY, repeat_rate=REPEAT_RATE): self.inputManager = InputManager(is_testing=is_testing, use_direct_keys=self.use_direct_keys, input_release_lag_ms=self.input_release_lag * 1000) self.mode = "regular" self.modeSwitcher = modeSwitcher self.detector = PatternDetector(self.patterns) self.pressed_keys = {} self.should_drag = False self.ctrlKey = False self.shiftKey = False self.altKey = False if( SPEECHREC_ENABLED == True and IS_WINDOWS == True ): from dragonfly import Grammar from lib.grammar.simple_grammar import SimpleSpeechCommand import pythoncom self.grammar = Grammar("Simple") self.simpleCommandRule = SimpleSpeechCommand(self.speech_commands, callback=self.toggle_speech) self.grammar.add_rule( self.simpleCommandRule ) def start( self ): update_overlay_image( "default" ) toggle_eyetracker() def exit( self ): if( self.mode == "speech" ): self.toggle_speech() update_overlay_image( "default" ) toggle_eyetracker() def handle_input( self, dataDicts ): self.detector.tick( dataDicts ) self.quadrant3x3 = self.detector.detect_mouse_quadrant( 3, 3 ) self.quadrant4x3 = self.detector.detect_mouse_quadrant( 4, 3 ) if( self.detect_silence() ): self.stop_drag_mouse() self.inputManager.release_non_toggle_keys() # Recognize speech commands in speech mode if( self.mode == "speech" ): if (IS_WINDOWS == True): pythoncom.PumpWaitingMessages() self.handle_speech( dataDicts ) else: print( "Speech recognition is only implemented for Windows") self.mode = "regular" # Regular quick command mode elif( self.mode == "regular" ): self.handle_sounds( dataDicts ) return self.detector.tickActions def handle_speech( self, dataDicts ): if (IS_WINDOWS == False): print( "No speech handler" ) return def handle_sounds( self, dataDicts ): print( "No sound handler" ) return # Toggle between variables # If the value is a list, turn them on in sequence after another def toggle( self, value ): if (isinstance(value, list)): turned_on_index = -1 for index, key in enumerate(value): if (key not in self.toggles): self.toggles[key] = False elif (self.toggles[key] == True): turned_on_index = index self.toggles[key] = False next_index = turned_on_index + 1 if (next_index >= len(value)): next_index = 0 self.toggles[value[next_index]] = True else: if (value not in self.toggles ): self.toggles[value] = False self.toggles[value] = not self.toggles[value] def enable( self, value ): if (isinstance(value, list)): for index, key in enumerate(value): self.toggles[key] = True else: self.toggles[value] = True def disable( self, value ): if (isinstance(value, list)): for index, key in enumerate(value): self.toggles[key] = False else: self.toggles[value] = False def detect( self, key ): if (key in self.toggles): return self.toggles[key] return self.detector.detect( key ) def detect_silence( self ): return self.detector.detect_silence() def drag_mouse( self ): self.toggle_drag_mouse( True ) def stop_drag_mouse( self ): self.toggle_drag_mouse( False ) def leftclick( self ): self.inputManager.click(button='left') def rightclick( self ): self.inputManager.click(button='right') def press( self, key ): self.inputManager.press( key ) def hold( self, key, repeat_rate_ms=0 ): self.inputManager.hold( key, repeat_rate_ms ) def release( self, key ): self.inputManager.release( key ) def release_special_keys( self ): self.inputManager.release_special_keys() def toggle_speech( self ): if( self.mode != "speech" ): self.mode = "speech" self.grammar.load() print( "--- TOGGLING SPEECH RECOGNITION ON" ) else: self.mode = "regular" self.grammar.unload() print( "--- TOGGLING SPEECH RECOGNITION OFF" ) toggle_speechrec() # Drag mouse for selection purposes def toggle_drag_mouse( self, should_drag ): if( self.should_drag != should_drag ): if( should_drag == True ): self.inputManager.mouseDown() else: self.inputManager.mouseUp() self.should_drag = should_drag # Detect when the cursor is inside an area def detect_inside_area( self, x, y, width, height ): return self.detector.detect_inside_minimap( x, y, width, height ) def update_overlay( self ): if( not( self.ctrlKey or self.shiftKey or self.altKey ) ): update_overlay_image( "default" ) else: modes = [] if( self.ctrlKey ): modes.append( "ctrl" ) if( self.shiftKey ): modes.append( "shift" ) if( self.altKey ): modes.append( "alt" ) update_overlay_image( "mode-%s" % ( "-".join( modes ) ) )
def dragonfly_recognise(spec, choices_values, timeout): """ Build a grammar based on spec and choices and send it to dragonfly """ # build the dragonfly request extras = [] for name, choices in choices_values.iteritems(): extras.append(Choice(name, dict((c, c) for c in choices))) results = Queue() class GrammarRule(CompoundRule): def _process_recognition(self, node, extras): logger.info('_process_recognition callback: %s', str(node)) results.put_nowait(Result(node=node, extras=extras)) def _process_begin(self): logger.debug('Rule:__process_begin') rule = GrammarRule(spec=spec, extras=extras) grammar = Grammar("grammar") grammar.add_rule(rule) # attach failure callback def process_recognition_failure(): logger.info('Grammar:process_recognition_failure') grammar.process_recognition_failure = process_recognition_failure grammar.load() if os.name is 'nt': winsound.PlaySound(data_path + "/grammar_loaded.wav", winsound.SND_ASYNC) logger.info("Grammar loaded: %s", spec) global CANCEL_FLAG CANCEL_FLAG = False future = time.time() + timeout while time.time() < future and results.empty(): if os.name is 'nt': pythoncom.PumpWaitingMessages() if CANCEL_FLAG: return {} time.sleep(.1) grammar.unload() try: result = results.get_nowait() except Empty: logger.info('No result, probably a timeout') return False if not results.empty(): raise Exception('Multiple results received') # filter all extras with _ because they are private return { "result": result.node.value(), "choices": {k: v for (k, v) in result.extras.items() if not k.startswith('_')} }
class TestRules(RuleTestCase): def setUp(self): RuleTestCase.setUp(self) engine = get_engine() if engine.name == 'natlink': # Stop Dragon from dictating text for the duration of these # tests. This is required when testing for mimic failures. self.temp_grammar = Grammar("temp") self.temp_grammar.add_rule(CompoundRule(spec="exclusive rule")) self.temp_grammar.load() self.temp_grammar.set_exclusiveness(True) def tearDown(self): RuleTestCase.tearDown(self) engine = get_engine() if engine.name == 'natlink': self.temp_grammar.set_exclusiveness(False) self.temp_grammar.unload() def process_grammars_context(self): engine = get_engine() if engine.name.startswith("sapi5"): engine.process_grammars_context() def test_multiple_rules(self): """ Verify that the engine successfully mimics each rule in a grammar with multiple rules. """ self.add_rule(CompoundRule(name="r1", spec="hello")) self.add_rule(CompoundRule(name="r2", spec="see you")) assert self.recognize_node("hello").words() == ["hello"] assert self.recognize_node("see you").words() == ["see", "you"] def test_rule_context(self): """ Verify that the engine works correctly with rule contexts. """ context = TestContext(True) self.add_rule( CompoundRule(name="r1", spec="test context", context=context)) self.grammar.load() # Test that the rule matches when in-context. results = self.recognize_node("test context").words() assert results == ["test", "context"] # Go out of context and test again. # Use the engine's mimic method because recognize_node won't return # RecognitionFailure like ElementTester.recognize does. context.active = False self.process_grammars_context() try: self.grammar.set_exclusiveness(True) self.assertRaises(MimicFailure, self.engine.mimic, "test context") finally: self.grammar.set_exclusiveness(False) # Test again after going back into context. context.active = True self.process_grammars_context() results = self.recognize_node("test context").words() assert results == ["test", "context"] def test_grammar_context(self): """ Verify that the engine works correctly with grammar contexts.""" # Recreate the RuleTestGrammar using a context and add a rule. context = TestContext(True) self.grammar = RuleTestGrammar(context=context) self.add_rule(CompoundRule(name="r1", spec="test context")) self.grammar.load() # Test that the rule matches when in-context. results = self.recognize_node("test context").words() assert results == ["test", "context"] # Go out of context and test again. context.active = False self.process_grammars_context() try: self.grammar.set_exclusiveness(True) self.assertRaises(MimicFailure, self.engine.mimic, "test context") finally: self.grammar.set_exclusiveness(False) # Test again after going back into context. context.active = True self.process_grammars_context() results = self.recognize_node("test context").words() assert results == ["test", "context"] def test_exclusive_grammars(self): """ Verify that the engine supports exclusive grammars. """ # This is here as grammar exclusivity is context related. # Set up two grammars to test with. class TestRule(CompoundRule): def __init__(self, spec): CompoundRule.__init__(self, spec=spec) self.words = None def _process_recognition(self, node, extras): self.words = self.spec grammar1 = Grammar(name="Grammar1") grammar1.add_rule(TestRule(spec="grammar one")) grammar2 = Grammar(name="Grammar2") grammar2.add_rule(TestRule(spec="grammar two")) grammar3 = Grammar(name="Grammar3") grammar3.add_rule(TestRule(spec="grammar three")) grammar1.load() grammar2.load() grammar3.load() # Set grammar1 as exclusive and make some assertions. grammar1.set_exclusiveness(True) self.engine.mimic("grammar one") assert grammar1.rules[0].words == "grammar one" self.assertRaises(MimicFailure, self.engine.mimic, "grammar two") # Set grammar2 as exclusive and make some assertions. # Both exclusive grammars should be active. grammar2.set_exclusiveness(True) self.engine.mimic("grammar one") assert grammar1.rules[0].words == "grammar one" self.engine.mimic("grammar two") assert grammar2.rules[0].words == "grammar two" # Non-exclusive grammar 'grammar3' should not be active. self.assertRaises(MimicFailure, self.engine.mimic, "grammar three") # Set both grammars as no longer exclusive and make some assertions. grammar1.set_exclusiveness(False) grammar2.set_exclusiveness(False) if get_engine().name == 'natlink': self.temp_grammar.set_exclusiveness(False) self.engine.mimic("grammar one") assert grammar1.rules[0].words == "grammar one" self.engine.mimic("grammar two") assert grammar2.rules[0].words == "grammar two" self.engine.mimic("grammar three") assert grammar3.rules[0].words == "grammar three"
def unload(self): for rule in self._rules: # unregister to prevent multiply registered rules during restart rule.deactivate() Grammar.unload(self)
class ExcelMode: def __init__(self, modeSwitcher, file): self.mode = "regular" self.modeSwitcher = modeSwitcher self.file = file self.grammar = Grammar("Excel") columnNumberRule = ColumnNumberPrintRule() self.grammar.add_rule(columnNumberRule) columnModeRule = ColumnModePrintRule() self.grammar.add_rule(columnModeRule) correctionRule = CorrectionRule() self.grammar.add_rule(correctionRule) copyRowRule = CopyRowRule() self.grammar.add_rule(copyRowRule) nextRowRule = MoveRule() self.grammar.add_rule(nextRowRule) pasteRule = PasteRule() self.grammar.add_rule(pasteRule) self.grammar.add_rule(ShiftRule()) def start(self): self.grammar.load() self.mode = "speech" toggle_speechrec() self.centerXPos, self.centerYPos = pyautogui.position() toggle_eyetracker() mute_sound() self.open_file() def handle_input(self, dataDicts): if (loud_detection(dataDicts, "bell")): self.modeSwitcher.turnOnModeSwitch() if (self.mode == "regular"): if (percentage_detection(dataDicts, "whistle", 75)): self.mode = "speech" toggle_speechrec() else: if (single_tap_detection(dataDicts, "cluck", 35, 1000)): click() elif (single_tap_detection(dataDicts, "fingersnap", 50, 1000)): click(button='right') elif (loud_detection(dataDicts, "sound_f")): scroll(150) elif (loud_detection(dataDicts, "sound_s")): scroll(-150) if (percentage_detection(dataDicts, "sound_thr", 75)): quadrant = detect_mouse_quadrant(3, 3) if (quadrant > 3): self.modeSwitcher.switchMode('browse') elif (self.mode == "speech"): self.speech_mode() if (percentage_detection(dataDicts, "sound_thr", 75)): self.mode = "regular" toggle_speechrec() def speech_mode(self): pythoncom.PumpWaitingMessages() time.sleep(.1) def open_file(self): if (self.file != ''): call(["start", self.file], shell=True) def exit(self): if (self.mode == "speech"): toggle_speechrec() self.mode = "regular" toggle_eyetracker() self.grammar.unload()
class MasterGrammar(object): """A MasterGrammar is built up from a specific set of active rules. They synthesize the different rule types into one dragonfly grammar. There is only ever one master grammar active at a time.""" def __init__(self, baseRuleSet, client, ruleCache): self.client = client self.ruleCache = ruleCache # Hashes that are directly part of this grammar self.baseRuleSet = set(baseRuleSet) # Hashes of rules that we discover are dependencies # of the base rule set self.dependencyRuleSet = set() # hash -> dragonfly rule self.concreteRules = {} # one hash per merge group, hash is of hashes of rules that were merged self.seriesRules = set() # one hash, hash is of hashes of rules that were merged self.terminatorRule = "" # one hash per rule, hash is the rule's actual hash self.independentRules = set() # Rule references are stored as hashes, so rules that # contain rule refs already effectively include those # rules in their hash, so just hashing the base set is # all we need. x = hashlib.sha256() x.update("".join(sorted([r for r in self.baseRuleSet]))) self.hash = x.hexdigest()[:32] # Hashes of rules we depend on but haven't arrived yet. # These will be discovered during the dfly grammar building # process. self.missing = set() self.checkDeps(self.fullRullSet) # build self.missing self.finalDflyRule = None self.dflyGrammar = None # word lists are *not* hashed. they are global state the # client can update at any time, and the change has to be # propogated into the currently active grammar. the client # can choose to make them rule specific by making the name # be the hash of the rule the word list applies to, but this # is only convention and not enforced self.concreteWordLists = {} @property def fullRullSet(self): return self.baseRuleSet | self.dependencyRuleSet def satisfyDependency(self, r): """Marks dependency on hash r as satisfied, and tries to build if no more known deps are missing. During the build process new indirect dependencies may still be discovered however.""" assert r in self.missing self.missing.remove(r) if not self.missing: self.build() def checkDep(self, r): "Checks if dep r is present. Not recursive." if r not in self.ruleCache: self.ruleCache[r] = NeedDependency() if isinstance(self.ruleCache[r], NeedDependency): self.ruleCache[r].add(self.hash) self.missing.add(r) return False return True def checkMissing(self): if self.missing: raise MissingDependency(copy(self.missing)) def checkDeps(self, ruleSet): "Recursively check if all deps in ruleSet are satisfied." if not ruleSet: return True newDeps = set() for r in ruleSet: if self.checkDep(r): rule = self.ruleCache[r] # HashedRule rule = rule.rule log.info("rule [%s]" % (rule, )) for e in rule.extras: if hasattr(e, "rule_ref"): newDeps.add(e.rule_ref) self.dependencyRuleSet.update(newDeps) self.checkDeps(newDeps) def ready(self): return len(self.missing) == 0 def build(self): if self.dflyGrammar: # already built return buildStartTime = time.time() self.checkMissing() self.checkDeps(self.fullRullSet) self.checkMissing() # from here on we assume all deps are present all the way down seriesGroups = {} terminal = {} allRules = [] mergeStartTime = time.time() # Merge series and terminal rules, set independent rules aside self.fullName = [] for r in self.fullRullSet: rule = self.ruleCache[r].rule hash = self.ruleCache[r].hash if rule.ruleType == RuleType.SERIES: if rule.seriesMergeGroup not in seriesGroups: seriesGroups[rule.seriesMergeGroup] = {} x = seriesGroups[rule.seriesMergeGroup] elif rule.ruleType == RuleType.TERMINAL: x = terminal elif rule.ruleType == RuleType.INDEPENDENT: x = {} if "mapping" not in x: x["mapping"] = {} if "extras" not in x: x["extras"] = {} if "defaults" not in x: x["defaults"] = {} if "name" not in x: x["name"] = "" if "hash" not in x: x["hash"] = set() x["ruleType"] = rule.ruleType x["seriesMergeGroup"] = rule.seriesMergeGroup x["name"] = x["name"] + ("," if x["name"] else "") + rule.name x["mapping"].update(rule.mapping.items()) for e in rule.extras: x["extras"][e.name] = e x["defaults"].update(rule.defaults.items()) log.info("Adding hash [%s] to name [%s]" % (hash, x["name"])) x["hash"].add(hash) x["built"] = False x["exported"] = (rule.ruleType == RuleType.INDEPENDENT) # allRules will contain all the rules we have left # *after* merging. So only one series rule per merge # group and only one terminal rule. allRules.append(x) mergeEndTime = time.time() log.info("Grammar merge time: %ss" % (mergeEndTime - mergeStartTime)) # We really should be doing a topological sort, but this # isn't a frequent operation so this inefficiency should # be OK. Keep trying to link deps until they're all good. uniqueRules = [] for r in allRules: if r not in uniqueRules: uniqueRules.append(r) self.fullName.append(r["name"]) self.fullName = ",".join(self.fullName) allRules = uniqueRules # collapse the hashes for r in allRules: assert type(r["hash"]) == set assert len(r["hash"]) >= 1 if r["ruleType"] in (RuleType.SERIES, RuleType.TERMINAL): # We generate a composite hash for our new composite rules log.info("Multi-hash: [%s]" % r["hash"]) hashes = sorted(list(r["hash"])) x = hashlib.sha256() x.update("".join(sorted([h for h in hashes]))) hash = x.hexdigest()[:32] log.info("Composite: [%s]" % hash) else: # We just use the exising hash for a rule if it's not composite [hash] = r["hash"] log.info("Single hash: [%s]" % r["hash"]) r["hash"] = hash allPrototypes = {i["hash"]: i for i in allRules} self.concreteTime = 0 cleanupTime = 0 for k, v in allPrototypes.items(): if not v["built"]: cleanupStart = time.time() self.cleanupProtoRule(v, allPrototypes) cleanupEnd = time.time() cleanupTime += (cleanupEnd - cleanupStart) log.info("Total Cleanup time: %ss" % cleanupTime) log.info("Total Concrete time: %ss" % (self.concreteTime)) #log.info("made it out of loop") self.buildFinalMergedRule() buildEndTime = time.time() log.info("Grammar build time: %ss" % (buildEndTime - buildStartTime)) self.setupFinalDflyGrammar() def buildFinalMergedRule(self): #log.info("Building final merged rule.") if not self.seriesRules and not self.terminatorRule: return extras = [] seriesRefNames = [] for i, r in enumerate(self.seriesRules): name = "s" + str(i) seriesRefNames.append(name) ref = dfly.RuleRef(self.concreteRules[r], name) extras.append(ref) seriesPart = "[" + " | ".join([("<" + r + ">") for r in seriesRefNames]) + "]" terminatorPart = "" if self.terminatorRule: extras.append( dfly.RuleRef(self.concreteRules[self.terminatorRule], "terminator")) terminatorPart = " [<terminator>]" masterPhrase = seriesPart + terminatorPart mapping = { masterPhrase: ReportingAction(masterPhrase, self.client, self.hash) } log.info( "Building master grammar rule with name [%s] mapping [%s] extras [%s] defaults [%s]" % (self.fullName, mapping, extras, {})) masterTimeStart = time.time() self.finalDflyRule = MappingRule(name=self.hash, mapping=mapping, extras=extras, defaults={}, exported=True) masterTimeEnd = time.time() log.info("Master rule construction time: %ss" % (masterTimeEnd - masterTimeStart)) def setupFinalDflyGrammar(self): log.info("Setting up final grammar.") assert not self.dflyGrammar self.dflyGrammar = Grammar(self.fullName + "Grammar") if self.finalDflyRule: self.dflyGrammar.add_rule(self.finalDflyRule) for r in self.independentRules: self.dflyGrammar.add_rule(self.concreteRules[r]) loadStart = time.time() self.dflyGrammar.load() loadEnd = time.time() log.info("Grammar load time: %ss" % (loadEnd - loadStart)) get_engine().set_exclusiveness(self.dflyGrammar, 1) # These should never be recognized on their own, only as part of the # master rule, quirk of dragonfly that you have to do this even though # they're only pulled in by ruleref. for r in self.seriesRules: self.concreteRules[r].disable() if self.terminatorRule: self.concreteRules[self.terminatorRule].disable() # independent rules only enabled via being a dependency need to have disable # called on their dragonfly version so that they don't get recognized by # themselves, same quirk. notEnabledRules = self.dependencyRuleSet - self.baseRuleSet for r in notEnabledRules: self.concreteRules[r].disable() # they're enabled by default, don't activate until explicitly made to self.dflyGrammar.disable() def active(self): #log.info("active check [%s %s %s]" % (self.dflyGrammar is None, self.dflyGrammar and self.dflyGrammar.loaded, self.dflyGrammar and self.dflyGrammar.enabled)) return self.dflyGrammar and self.dflyGrammar.loaded and self.dflyGrammar.enabled def activate(self): self.build() self.dflyGrammar.enable() log.info("Grammar activated: [%s]" % self.hash) def deactivate(self): # it's possible we never built successfully if self.dflyGrammar: self.dflyGrammar.disable() log.info("Grammar deactivated: [%s]" % self.hash) def unload(self): self.deactivate() if self.dflyGrammar: self.dflyGrammar.unload() def buildConcreteRule(self, r): # for independent rules we could use the plain # name, but it turns out Dragon crashes if your # names get too long, so for combined rules we # just use the hash as the name... hopefully # that's under the limit name = r["hash"] if r["ruleType"] == RuleType.SERIES: t = SeriesMappingRule elif r["ruleType"] == RuleType.TERMINAL: t = MappingRule else: t = MappingRule constructionStartTime = time.time() log.info( "Building rule [%s] with size [%s] num extras [%s] num defaults [%s]" % (r["name"], len(r["mapping"]), len( r["extras"]), len(r["defaults"]))) rule = t(name=name, mapping=r["mapping"], extras=r["extras"], defaults=r["defaults"], exported=r["exported"]) constructionEndTime = time.time() log.info("Rule construction time: %ss" % (constructionEndTime - constructionStartTime)) self.concreteRules[r["hash"]] = rule if r["ruleType"] == RuleType.SERIES: self.seriesRules.add(r["hash"]) elif r["ruleType"] == RuleType.TERMINAL: self.terminatorRule = r["hash"] elif r["ruleType"] == RuleType.INDEPENDENT: self.independentRules.add(r["hash"]) else: assert False log.info("done building") def cleanupProtoRule(self, r, allPrototypes): # have to uniquify in this round about way because lists # aren't hashable and we need them for ListRef. if type(r["extras"]) == dict: r["extras"] = r["extras"].values() newExtras = [] for e in r["extras"]: if isinstance(e, protocol.Integer): newExtras.append(dfly.Integer(e.name, e.min, e.max)) elif isinstance(e, protocol.Dictation): newExtras.append(dfly.Dictation(e.name)) elif isinstance(e, protocol.Repetition): if e.rule_ref not in self.concreteRules: self.cleanupProtoRule(allPrototypes[e.rule_ref], allPrototypes) # Dragonfly wants RuleRef to take a RuleRef rather than an actual # Rule, so we just make one rather than forcing the server to # handle this, see protocol.py comments. concrete = self.concreteRules[e.rule_ref] log.info("concrete type: [%s]" % type(concrete)) newExtras.append( dfly.Repetition(dfly.RuleRef(rule=concrete), e.min, e.max, e.name)) elif isinstance(e, protocol.RuleRef): if e.rule_ref not in self.concreteRules: self.cleanupProtoRule(allPrototypes[e.rule_ref], allPrototypes) newExtras.append( dfly.RuleRef(self.concreteRules[e.rule_ref], e.name)) elif isinstance(e, protocol.ListRef): self.concreteWordLists[e.name] = List(e.name + "ConcreteList") # self.concreteWordLists[e.name].set(e.words) newExtras.append( dfly.ListRef(e.ref_name, self.concreteWordLists[e.name])) else: raise Exception("Unknown extra type: [%s]" % e) r["extras"] = newExtras self.concreteStartTime = time.time() self.buildConcreteRule(r) self.concreteEndTime = time.time() self.concreteTime += (self.concreteEndTime - self.concreteStartTime) r["built"] = True return True def updateWordList(self, name, words): if name not in self.concreteWordLists: # log.info("Word list [%s] not in grammar [%s], ignoring" % (name, self.hash)) return # We want to check if the value has actually changed because List's # set method will blindly tell Dragon to delete its old list and replace # it with this one and we don't want to disturb Dragon unless we have to # because Dragon is slow. if sorted(words) != sorted(self.concreteWordLists[name]): log.info( "Updating word list [%s] on grammar [%s] with contents [%s]" % (name, self.hash, len(words))) log.info("old list: %s" % self.concreteWordLists[name]) # TODO: need to check existing load state, then send a loading message here, then restore # old state. This way we can see when word lists are taking a long time to load... updateStart = time.time() self.concreteWordLists[name].set(words) updateEnd = time.time() log.info("Word list update time: %ss" % (updateEnd - updateStart))
class MasterGrammar(object): """A MasterGrammar is built up from a specific set of active rules. They synthesize the different rule types into one dragonfly grammar. There is only ever one master grammar active at a time.""" def __init__(self, baseRuleSet, client, ruleCache): self.client = client self.ruleCache = ruleCache # Hashes that are directly part of this grammar self.baseRuleSet = set(baseRuleSet) # Hashes of rules that we discover are dependencies # of the base rule set self.dependencyRuleSet = set() # hash -> dragonfly rule self.concreteRules = {} # one hash per merge group, hash is of hashes of rules that were merged self.seriesRules = set() # one hash, hash is of hashes of rules that were merged self.terminatorRule = "" # one hash per rule, hash is the rule's actual hash self.independentRules = set() # Rule references are stored as hashes, so rules that # contain rule refs already effectively include those # rules in their hash, so just hashing the base set is # all we need. x = hashlib.sha256() x.update("".join(sorted([r for r in self.baseRuleSet]))) self.hash = x.hexdigest()[:32] # Hashes of rules we depend on but haven't arrived yet. # These will be discovered during the dfly grammar building # process. self.missing = set() self.checkDeps(self.fullRullSet) # build self.missing self.finalDflyRule = None self.dflyGrammar = None # word lists are *not* hashed. they are global state the # client can update at any time, and the change has to be # propogated into the currently active grammar. the client # can choose to make them rule specific by making the name # be the hash of the rule the word list applies to, but this # is only convention and not enforced self.concreteWordLists = {} @property def fullRullSet(self): return self.baseRuleSet | self.dependencyRuleSet def satisfyDependency(self, r): """Marks dependency on hash r as satisfied, and tries to build if no more known deps are missing. During the build process new indirect dependencies may still be discovered however.""" assert r in self.missing self.missing.remove(r) if not self.missing: self.build() def checkDep(self, r): "Checks if dep r is present. Not recursive." if r not in self.ruleCache: self.ruleCache[r] = NeedDependency() if isinstance(self.ruleCache[r], NeedDependency): self.ruleCache[r].add(self.hash) self.missing.add(r) return False return True def checkMissing(self): if self.missing: raise MissingDependency(copy(self.missing)) def checkDeps(self, ruleSet): "Recursively check if all deps in ruleSet are satisfied." if not ruleSet: return True newDeps = set() for r in ruleSet: if self.checkDep(r): rule = self.ruleCache[r] # HashedRule rule = rule.rule log.info("rule [%s]" % (rule,)) for e in rule.extras: if hasattr(e, "rule_ref"): newDeps.add(e.rule_ref) self.dependencyRuleSet.update(newDeps) self.checkDeps(newDeps) def ready(self): return len(self.missing) == 0 def build(self): if self.dflyGrammar: # already built return buildStartTime = time.time() self.checkMissing() self.checkDeps(self.fullRullSet) self.checkMissing() # from here on we assume all deps are present all the way down seriesGroups = {} terminal = {} allRules = [] mergeStartTime = time.time() # Merge series and terminal rules, set independent rules aside self.fullName = [] for r in self.fullRullSet: rule = self.ruleCache[r].rule hash = self.ruleCache[r].hash if rule.ruleType == RuleType.SERIES: if rule.seriesMergeGroup not in seriesGroups: seriesGroups[rule.seriesMergeGroup] = {} x = seriesGroups[rule.seriesMergeGroup] elif rule.ruleType == RuleType.TERMINAL: x = terminal elif rule.ruleType == RuleType.INDEPENDENT: x = {} if "mapping" not in x: x["mapping"] = {} if "extras" not in x: x["extras"] = {} if "defaults" not in x: x["defaults"] = {} if "name" not in x: x["name"] = "" if "hash" not in x: x["hash"] = set() x["ruleType"] = rule.ruleType x["seriesMergeGroup"] = rule.seriesMergeGroup x["name"] = x["name"] + ("," if x["name"] else "") + rule.name x["mapping"].update(rule.mapping.items()) for e in rule.extras: x["extras"][e.name] = e x["defaults"].update(rule.defaults.items()) log.info("Adding hash [%s] to name [%s]" % (hash, x["name"])) x["hash"].add(hash) x["built"] = False x["exported"] = (rule.ruleType == RuleType.INDEPENDENT) # allRules will contain all the rules we have left # *after* merging. So only one series rule per merge # group and only one terminal rule. allRules.append(x) mergeEndTime = time.time() log.info("Grammar merge time: %ss" % (mergeEndTime - mergeStartTime)) # We really should be doing a topological sort, but this # isn't a frequent operation so this inefficiency should # be OK. Keep trying to link deps until they're all good. uniqueRules = [] for r in allRules: if r not in uniqueRules: uniqueRules.append(r) self.fullName.append(r["name"]) self.fullName = ",".join(self.fullName) allRules = uniqueRules # collapse the hashes for r in allRules: assert type(r["hash"]) == set assert len(r["hash"]) >= 1 if r["ruleType"] in (RuleType.SERIES, RuleType.TERMINAL): # We generate a composite hash for our new composite rules log.info("Multi-hash: [%s]" % r["hash"]) hashes = sorted(list(r["hash"])) x = hashlib.sha256() x.update("".join(sorted([h for h in hashes]))) hash = x.hexdigest()[:32] log.info("Composite: [%s]" % hash) else: # We just use the exising hash for a rule if it's not composite [hash] = r["hash"] log.info("Single hash: [%s]" % r["hash"]) r["hash"] = hash allPrototypes = { i["hash"] : i for i in allRules } self.concreteTime = 0 cleanupTime = 0 for k, v in allPrototypes.items(): if not v["built"]: cleanupStart = time.time() self.cleanupProtoRule(v, allPrototypes) cleanupEnd = time.time() cleanupTime += (cleanupEnd - cleanupStart) log.info("Total Cleanup time: %ss" % cleanupTime) log.info("Total Concrete time: %ss" % (self.concreteTime)) #log.info("made it out of loop") self.buildFinalMergedRule() buildEndTime = time.time() log.info("Grammar build time: %ss" % (buildEndTime - buildStartTime)) self.setupFinalDflyGrammar() def buildFinalMergedRule(self): #log.info("Building final merged rule.") if not self.seriesRules and not self.terminatorRule: return extras = [] seriesRefNames = [] for i, r in enumerate(self.seriesRules): name = "s" + str(i) seriesRefNames.append(name) ref = dfly.RuleRef(self.concreteRules[r], name) extras.append(ref) seriesPart = "[" + " | ".join([("<" + r + ">") for r in seriesRefNames]) + "]" terminatorPart = "" if self.terminatorRule: extras.append(dfly.RuleRef(self.concreteRules[self.terminatorRule], "terminator")) terminatorPart = " [<terminator>]" masterPhrase = seriesPart + terminatorPart mapping = { masterPhrase : ReportingAction(masterPhrase, self.client, self.hash) } log.info("Building master grammar rule with name [%s] mapping [%s] extras [%s] defaults [%s]" % (self.fullName, mapping, extras, {})) masterTimeStart = time.time() self.finalDflyRule = MappingRule(name=self.hash, mapping=mapping, extras=extras, defaults={}, exported=True) masterTimeEnd = time.time() log.info("Master rule construction time: %ss" % (masterTimeEnd - masterTimeStart)) def setupFinalDflyGrammar(self): log.info("Setting up final grammar.") assert not self.dflyGrammar self.dflyGrammar = Grammar(self.fullName + "Grammar") if self.finalDflyRule: self.dflyGrammar.add_rule(self.finalDflyRule) for r in self.independentRules: self.dflyGrammar.add_rule(self.concreteRules[r]) loadStart = time.time() self.dflyGrammar.load() loadEnd = time.time() log.info("Grammar load time: %ss" % (loadEnd - loadStart)) get_engine().set_exclusiveness(self.dflyGrammar, 1) # These should never be recognized on their own, only as part of the # master rule, quirk of dragonfly that you have to do this even though # they're only pulled in by ruleref. for r in self.seriesRules: self.concreteRules[r].disable() if self.terminatorRule: self.concreteRules[self.terminatorRule].disable() # independent rules only enabled via being a dependency need to have disable # called on their dragonfly version so that they don't get recognized by # themselves, same quirk. notEnabledRules = self.dependencyRuleSet - self.baseRuleSet for r in notEnabledRules: self.concreteRules[r].disable() # they're enabled by default, don't activate until explicitly made to self.dflyGrammar.disable() def active(self): #log.info("active check [%s %s %s]" % (self.dflyGrammar is None, self.dflyGrammar and self.dflyGrammar.loaded, self.dflyGrammar and self.dflyGrammar.enabled)) return self.dflyGrammar and self.dflyGrammar.loaded and self.dflyGrammar.enabled def activate(self): self.build() self.dflyGrammar.enable() log.info("Grammar activated: [%s]" % self.hash) def deactivate(self): # it's possible we never built successfully if self.dflyGrammar: self.dflyGrammar.disable() log.info("Grammar deactivated: [%s]" % self.hash) def unload(self): self.deactivate() if self.dflyGrammar: self.dflyGrammar.unload() def buildConcreteRule(self, r): # for independent rules we could use the plain # name, but it turns out Dragon crashes if your # names get too long, so for combined rules we # just use the hash as the name... hopefully # that's under the limit name = r["hash"] if r["ruleType"] == RuleType.SERIES: t = SeriesMappingRule elif r["ruleType"] == RuleType.TERMINAL: t = MappingRule else: t = MappingRule constructionStartTime = time.time() log.info("Building rule [%s] with size [%s] num extras [%s] num defaults [%s]" % (r["name"], len(r["mapping"]), len(r["extras"]), len(r["defaults"]))) rule = t(name=name, mapping=r["mapping"], extras=r["extras"], defaults=r["defaults"], exported=r["exported"]) constructionEndTime = time.time() log.info("Rule construction time: %ss" % (constructionEndTime - constructionStartTime)) self.concreteRules[r["hash"]] = rule if r["ruleType"] == RuleType.SERIES: self.seriesRules.add(r["hash"]) elif r["ruleType"] == RuleType.TERMINAL: self.terminatorRule = r["hash"] elif r["ruleType"] == RuleType.INDEPENDENT: self.independentRules.add(r["hash"]) else: assert False log.info("done building") def cleanupProtoRule(self, r, allPrototypes): # have to uniquify in this round about way because lists # aren't hashable and we need them for ListRef. if type(r["extras"]) == dict: r["extras"] = r["extras"].values() newExtras = [] for e in r["extras"]: if isinstance(e, protocol.Integer): newExtras.append(dfly.Integer(e.name, e.min, e.max)) elif isinstance(e, protocol.Dictation): newExtras.append(dfly.Dictation(e.name)) elif isinstance(e, protocol.Repetition): if e.rule_ref not in self.concreteRules: self.cleanupProtoRule(allPrototypes[e.rule_ref], allPrototypes) # Dragonfly wants RuleRef to take a RuleRef rather than an actual # Rule, so we just make one rather than forcing the server to # handle this, see protocol.py comments. concrete = self.concreteRules[e.rule_ref] log.info("concrete type: [%s]" % type(concrete)) newExtras.append(dfly.Repetition(dfly.RuleRef(rule=concrete), e.min, e.max, e.name)) elif isinstance(e, protocol.RuleRef): if e.rule_ref not in self.concreteRules: self.cleanupProtoRule(allPrototypes[e.rule_ref], allPrototypes) newExtras.append(dfly.RuleRef(self.concreteRules[e.rule_ref], e.name)) elif isinstance(e, protocol.ListRef): self.concreteWordLists[e.name] = List(e.name + "ConcreteList") # self.concreteWordLists[e.name].set(e.words) newExtras.append(dfly.ListRef(e.ref_name, self.concreteWordLists[e.name])) else: raise Exception("Unknown extra type: [%s]" % e) r["extras"] = newExtras self.concreteStartTime = time.time() self.buildConcreteRule(r) self.concreteEndTime = time.time() self.concreteTime += (self.concreteEndTime - self.concreteStartTime) r["built"] = True return True def updateWordList(self, name, words): if name not in self.concreteWordLists: # log.info("Word list [%s] not in grammar [%s], ignoring" % (name, self.hash)) return # We want to check if the value has actually changed because List's # set method will blindly tell Dragon to delete its old list and replace # it with this one and we don't want to disturb Dragon unless we have to # because Dragon is slow. if sorted(words) != sorted(self.concreteWordLists[name]): log.info("Updating word list [%s] on grammar [%s] with contents [%s]" % (name, self.hash, len(words))) log.info("old list: %s" % self.concreteWordLists[name]) # TODO: need to check existing load state, then send a loading message here, then restore # old state. This way we can see when word lists are taking a long time to load... updateStart = time.time() self.concreteWordLists[name].set(words) updateEnd = time.time() log.info("Word list update time: %ss" % (updateEnd - updateStart))
from dragonfly.engines.backend_sapi5.engine import Sapi5InProcEngine from dragonfly import (Grammar, CompoundRule, Dictation, Choice) except: error("Failed to import dragonfly, path: %s" % dragonfly_path) engine = Sapi5InProcEngine() engine.connect() engine.speak('Speak recognition active!') # Voice command rule combining spoken form and recognition processing. class ExampleRule(CompoundRule): spec = "do something computer" # Spoken form of command. def _process_recognition(self, node, extras): # Callback when command is spoken. print "Voice command spoken." # Create a grammar which contains and loads the command rule. grammar = Grammar("example grammar") # Create a grammar to contain the command rule. grammar.add_rule(ExampleRule()) # Add the command rule to the grammar. logger.info("Loading Grammar") grammar.load() # Load the grammar. logger.info("Grammar loaded") while True: pythoncom.PumpWaitingMessages() sleep(.1) grammar.unload()