def generate(self, message: str, doc: Doc = None, ignore_topics: List[str] = []) -> Optional[str]: if doc is None: filtered_message = MarkovFilters.filter_input(message) doc = self._nlp(filtered_message) subjects = [] for token in doc: if (token.text in ignore_topics): continue markov_word = self._markov_model.select(token.text) if markov_word is not None: subjects.append(markov_word) if len(subjects) == 0: UNHEARD_LIST = [ "Didn’t catch that", "Try again", "Are you even trying", "That might be too much for me right now", "I’ll learn how eventually", "I don't know how to respond to that yet" ] UNHEARD_RESPONSE = random.choice(UNHEARD_LIST) return UNHEARD_RESPONSE def structure_generator(): sentence_stats_manager = InputTextStatManager() while True: choices, p_values = sentence_stats_manager.probabilities() if len(choices) > 0: num_sentences = np.random.choice(choices, p=p_values) else: num_sentences = np.random.randint(1, 5) yield self._structure_scheduler.predict( num_sentences=num_sentences) generator = MarkovGenerator(structure_generator=structure_generator(), subjects=subjects) reply_words = [] sentences = generator.generate(db=self._markov_model) if sentences is None: MISUNDERSTOOD_LIST = [ 'Huh.', 'Huh', 'Huh!', 'Huh?', 'Huh!?', 'HUH?' ] MISUNDERSTOOD_REPONSE = random.choice(MISUNDERSTOOD_LIST) return MISUNDERSTOOD_REPONSE for sentence in sentences: for word_idx, word in enumerate(sentence): if not word.compound: text = CapitalizationMode.transform(word.mode, word.text) else: text = word.text reply_words.append(text) reply = " ".join(reply_words) filtered_reply = MarkovFilters.smooth_output(reply) return filtered_reply
def from_token(token: Token) -> 'MarkovWord': if CapitalizationMode.from_token( token, CAPITALIZATION_COMPOUND_RULES) == CapitalizationMode.COMPOUND: compound = True else: compound = False return MarkovWord(token.text, Pos.from_token(token), compound=compound, neighbors={})
def from_token(token: Token) -> 'MarkovNeighbor': key = token.text.lower() text = token.text if CapitalizationMode.from_token(token, CAPITALIZATION_COMPOUND_RULES) == CapitalizationMode.COMPOUND: compound = True else: compound = False pos = Pos.from_token(token) values = [0, 0] dist = [0] * (MARKOV_WINDOW_SIZE * 2 + 1) return MarkovNeighbor(key, text, pos, compound, values, dist)
def from_token(token: Token) -> 'MarkovNeighbor': key = token.text.lower() text = token.text if CapitalizationMode.from_token(token, CAPITALIZATION_COMPOUND_RULES) == CapitalizationMode.COMPOUND: compound = True else: compound = False pos = Pos.from_token(token) values = [0, 0] dist = [0] * (MARKOV_WINDOW_SIZE * 2 + 1) return MarkovNeighbor(key, text, pos, compound, values, dist)
def preprocess(self, doc: Doc) -> bool: if len(self.data) >= STRUCTURE_MODEL_TRAINING_MAX_SIZE: return False sequence = [] previous_item = None for sentence_idx, sentence in enumerate(doc.sents): if len(self.data) >= STRUCTURE_MODEL_TRAINING_MAX_SIZE: return False for token_idx, token in enumerate(sentence): item = StructureFeatureAnalyzer.analyze( token, CapitalizationMode.from_token( token, CAPITALIZATION_COMPOUND_RULES)) label = item if len(sequence) == 0: # Offset data by one, making label point to the next data item sequence.append( PoSCapitalizationMode( Pos.NONE, CapitalizationMode.NONE).to_embedding()) else: sequence.append(previous_item) # We only want the latest SEQUENCE_LENGTH items sequence = sequence[-StructureModel.SEQUENCE_LENGTH:] self.data.append(sequence.copy()) self.labels.append(label) previous_item = item # Handle EOS after each sentence item = PoSCapitalizationMode( Pos.EOS, CapitalizationMode.NONE).to_embedding() label = item sequence.append(previous_item) # We only want the latest SEQUENCE_LENGTH items sequence = sequence[-StructureModel.SEQUENCE_LENGTH:] self.data.append(sequence.copy()) self.labels.append(label) previous_item = item return True
def main(): np.random.seed(int(time.time())) markov_db = MarkovTrieDb(MARKOV_DB_PATH) structure_model = StructureModelScheduler(use_gpu=USE_GPU) structure_model.start() structure_model.load(STRUCTURE_MODEL_PATH) subjects = [] for word in ['Some', 'Words', 'Here']: select_word = markov_db.select(word) if select_word is not None: subjects.append(select_word) else: print("Couldn't select %s" % word) for i in range(0, 1000): def structure_generator(): while True: yield structure_model.predict(num_sentences=1) markov_generator = MarkovGenerator(structure_generator(), subjects) words = [] sentences = markov_generator.generate(markov_db) if sentences is None: continue for sentence_idx, sentence in enumerate(sentences): pos_list = [word.pos for word in sentence] for word_idx, word in enumerate(sentence): if not word.compound: text = CapitalizationMode.transform( word.mode, sentences[sentence_idx][word_idx].text, ) else: text = word.text words.append(text) message = " ".join(words) message = MarkovFilters.smooth_output(message) print(message)
def generate(self, message: str, doc: Doc = None, ignore_topics: List[str] = []) -> Optional[str]: if doc is None: filtered_message = MarkovFilters.filter_input(message) doc = self._nlp(filtered_message) subjects = [] for token in doc: if(token.text in ignore_topics): continue markov_word = self._markov_model.select(token.text) if markov_word is not None: subjects.append(markov_word) if len(subjects) == 0: return "I wasn't trained on that!" def structure_generator(): sentence_stats_manager = InputTextStatManager() while True: choices, p_values = sentence_stats_manager.probabilities() if len(choices) > 0: num_sentences = np.random.choice(choices, p=p_values) else: num_sentences = np.random.randint(1, 5) yield self._structure_scheduler.predict(num_sentences=num_sentences) generator = MarkovGenerator(structure_generator=structure_generator(), subjects=subjects) reply_words = [] sentences = generator.generate(db=self._markov_model) if sentences is None: return "Huh?" for sentence in sentences: for word_idx, word in enumerate(sentence): if not word.compound: text = CapitalizationMode.transform(word.mode, word.text) else: text = word.text reply_words.append(text) reply = " ".join(reply_words) filtered_reply = MarkovFilters.smooth_output(reply) return filtered_reply
def main(): np.random.seed(int(time.time())) markov_db = MarkovTrieDb(MARKOV_DB_PATH) structure_model = StructureModelScheduler(use_gpu=USE_GPU) structure_model.start() structure_model.load(STRUCTURE_MODEL_PATH) subjects = [] for word in ['Some', 'Words', 'Here']: select_word = markov_db.select(word) if select_word is not None: subjects.append(select_word) else: print("Couldn't select %s" % word) for i in range(0, 1000): def structure_generator(): while True: yield structure_model.predict(num_sentences=1) markov_generator = MarkovGenerator(structure_generator(), subjects) words = [] sentences = markov_generator.generate(markov_db) if sentences is None: continue for sentence_idx, sentence in enumerate(sentences): pos_list = [word.pos for word in sentence] for word_idx, word in enumerate(sentence): if not word.compound: text = CapitalizationMode.transform(word.mode, sentences[sentence_idx][word_idx].text, ) else: text = word.text words.append(text) message = " ".join(words) message = MarkovFilters.smooth_output(message) print(message)
def preprocess(self, doc: Doc) -> bool: if len(self.data) >= STRUCTURE_MODEL_TRAINING_MAX_SIZE: return False sequence = [] previous_item = None for sentence_idx, sentence in enumerate(doc.sents): if len(self.data) >= STRUCTURE_MODEL_TRAINING_MAX_SIZE: return False for token_idx, token in enumerate(sentence): item = StructureFeatureAnalyzer.analyze( token, CapitalizationMode.from_token(token, CAPITALIZATION_COMPOUND_RULES)) label = item if len(sequence) == 0: # Offset data by one, making label point to the next data item sequence.append(PoSCapitalizationMode(Pos.NONE, CapitalizationMode.NONE).to_embedding()) else: sequence.append(previous_item) # We only want the latest SEQUENCE_LENGTH items sequence = sequence[-StructureModel.SEQUENCE_LENGTH:] self.data.append(sequence.copy()) self.labels.append(label) previous_item = item # Handle EOS after each sentence item = PoSCapitalizationMode(Pos.EOS, CapitalizationMode.NONE).to_embedding() label = item sequence.append(previous_item) # We only want the latest SEQUENCE_LENGTH items sequence = sequence[-StructureModel.SEQUENCE_LENGTH:] self.data.append(sequence.copy()) self.labels.append(label) previous_item = item return True
def from_token(token: Token) -> 'MarkovWord': if CapitalizationMode.from_token(token, CAPITALIZATION_COMPOUND_RULES) == CapitalizationMode.COMPOUND: compound = True else: compound = False return MarkovWord(token.text, Pos.from_token(token), compound=compound, neighbors={})
def analyze(token: Token, mode: CapitalizationMode): pos = Pos.from_token(token) mode = PoSCapitalizationMode(pos, mode) return mode.to_embedding()
def from_embedding(embedding: int): pos_part = int(embedding / len(CapitalizationMode)) mode_part = int(embedding % len(CapitalizationMode)) return PoSCapitalizationMode(Pos(pos_part), CapitalizationMode(mode_part))
def analyze(token: Token, mode: CapitalizationMode): pos = Pos.from_token(token) mode = PoSCapitalizationMode(pos, mode) return mode.to_embedding()