def expand_intents(self, include_additional_entities=False): # load entities first in the file and build a dictionary result = dict() entities_dict = dict() for intent_name, intent_file_path in self.get_intent_names(): for entity_type, entities_array in self.intent_training_file_content( intent_file_path, 'entities', False): entities_dict[entity_type] = entities_array # load intents again from file for intent_type, intent_array in self.intent_training_file_content( intent_file_path, 'intent'): intent_sentences = set() for line in intent_array: line_tokens = self._nlp.tokenization.tokenize(line) expanded = expand_parentheses(line_tokens) for sentence_tokens in expanded: sentence = self._nlp.tokenization.detokenize( sentence_tokens) fieldnames = [ fname for _, fname, _, _ in Formatter().parse(sentence) if fname ] fields_dict = dict() for fieldname in fieldnames: if fieldname in entities_dict: fields_dict[fieldname] = entities_dict[ fieldname].copy() else: if include_additional_entities: field_values = self.get_additional_entities( fieldname) if len(field_values) > 0: fields_dict[fieldname] = field_values if len(fields_dict) > 0: keys, values = zip(*fields_dict.items()) permutations = [ dict(zip(keys, v)) for v in itertools.product(*values) ] for p in permutations: entities_dict_permutation = EntitiesDict(p) intent_sentences.add( sentence.format( **entities_dict_permutation)) else: intent_sentences.add(sentence) result[intent_type] = list(intent_sentences) return result
def expand_options(parentheses_line: str) -> list: """ Convert 'test (a|b)' -> ['test a', 'test b'] Args: parentheses_line: Input line to expand Returns: List of expanded possibilities """ # 'a(this|that)b' -> [['a', 'this', 'b'], ['a', 'that', 'b']] options = expand_parentheses(re.split(r'([(|)])', parentheses_line)) return [re.sub(r'\s+', ' ', ' '.join(i)).strip() for i in options]
def expand_options(parentheses_line: str) -> list: """ Convert 'test (a|b)' -> ['test a', 'test b'] Args: parentheses_line: Input line to expand Returns: List of expanded possibilities """ # 'a(this|that)b' -> [['a', 'this', 'b'], ['a', 'that', 'b']] options = expand_parentheses(re.split(r'([(|)])', parentheses_line)) return [re.sub(r'\s+', ' ', ' '.join(i)).strip() for i in options]
def expand_keywords(sentece): kwords = {"required": [], "optional": []} in_optional = False for exp in expand_parentheses(tokenize(sentece)): if "[" in exp: in_optional = True required = exp[:exp.index("[")] kwords["required"].append(" ".join(required)) optional = exp[exp.index("[") + 1:] kwords["optional"].append(" ".join(optional).replace("]", "").strip()) elif in_optional: optional = exp kwords["optional"].append(" ".join(optional).replace("]", "").strip()) if "]" in exp: in_optional = False else: kwords["required"].append(" ".join(exp)) return kwords
def add_lines(self, name, lines): lines = remove_comments(lines) self.sent_lists[name] = sum([expand_parentheses(tokenize(line)) for line in lines], []) self.sent_lists[name] = [i for i in self.sent_lists[name] if i]
def xp(s): return {''.join(sent) for sent in expand_parentheses(tokenize(s))}
def expand_options(sentece): sentences = [] sentece = sentece.replace("[", "(").replace("]", "| )") for exp in expand_parentheses(tokenize(sentece)): sentences.append(" ".join(exp).replace("(", "[").replace(")", "]")) return sentences