def expand_intents(self, include_additional_entities=False):
# load entities first in the file and build a dictionary
result = dict()
entities_dict = dict()
for intent_name, intent_file_path in self.get_intent_names():
for entity_type, entities_array in self.intent_training_file_content(
intent_file_path, 'entities', False):
entities_dict[entity_type] = entities_array
# load intents again from file
for intent_type, intent_array in self.intent_training_file_content(
intent_file_path, 'intent'):
intent_sentences = set()
for line in intent_array:
line_tokens = self._nlp.tokenization.tokenize(line)
expanded = expand_parentheses(line_tokens)
for sentence_tokens in expanded:
sentence = self._nlp.tokenization.detokenize(
sentence_tokens)
fieldnames = [
fname
for _, fname, _, _ in Formatter().parse(sentence)
if fname
]
fields_dict = dict()
for fieldname in fieldnames:
if fieldname in entities_dict:
fields_dict[fieldname] = entities_dict[
fieldname].copy()
else:
if include_additional_entities:
field_values = self.get_additional_entities(
fieldname)
if len(field_values) > 0:
fields_dict[fieldname] = field_values
if len(fields_dict) > 0:
keys, values = zip(*fields_dict.items())
permutations = [
dict(zip(keys, v))
for v in itertools.product(*values)
]
for p in permutations:
entities_dict_permutation = EntitiesDict(p)
intent_sentences.add(
sentence.format(
**entities_dict_permutation))
else:
intent_sentences.add(sentence)
result[intent_type] = list(intent_sentences)
return result
def expand_options(parentheses_line: str) -> list:
"""
Convert 'test (a|b)' -> ['test a', 'test b']
Args:
parentheses_line: Input line to expand
Returns:
List of expanded possibilities
"""
# 'a(this|that)b' -> [['a', 'this', 'b'], ['a', 'that', 'b']]
options = expand_parentheses(re.split(r'([(|)])', parentheses_line))
return [re.sub(r'\s+', ' ', ' '.join(i)).strip() for i in options]
def expand_options(parentheses_line: str) -> list:
"""
Convert 'test (a|b)' -> ['test a', 'test b']
Args:
parentheses_line: Input line to expand
Returns:
List of expanded possibilities
"""
# 'a(this|that)b' -> [['a', 'this', 'b'], ['a', 'that', 'b']]
options = expand_parentheses(re.split(r'([(|)])', parentheses_line))
return [re.sub(r'\s+', ' ', ' '.join(i)).strip() for i in options]
def expand_keywords(sentece):
kwords = {"required": [], "optional": []}
in_optional = False
for exp in expand_parentheses(tokenize(sentece)):
if "[" in exp:
in_optional = True
required = exp[:exp.index("[")]
kwords["required"].append(" ".join(required))
optional = exp[exp.index("[") + 1:]
kwords["optional"].append(" ".join(optional).replace("]", "").strip())
elif in_optional:
optional = exp
kwords["optional"].append(" ".join(optional).replace("]", "").strip())
if "]" in exp:
in_optional = False
else:
kwords["required"].append(" ".join(exp))
return kwords
def add_lines(self, name, lines):
lines = remove_comments(lines)
self.sent_lists[name] = sum([expand_parentheses(tokenize(line)) for line in lines], [])
self.sent_lists[name] = [i for i in self.sent_lists[name] if i]
def xp(s):
return {''.join(sent) for sent in expand_parentheses(tokenize(s))}
def expand_options(sentece):
sentences = []
sentece = sentece.replace("[", "(").replace("]", "| )")
for exp in expand_parentheses(tokenize(sentece)):
sentences.append(" ".join(exp).replace("(", "[").replace(")", "]"))
return sentences