def run(self, template_filepath):
AST.reset_instance()
parser = Parser(template_filepath)
parser.parse()
self.generator = Generator()
self.train_examples = list(self.generator.generate_train())
self.test_examples = list(self.generator.generate_test())
def finish_execution(self):
for (unit_type, unit_name) in self._units_to_delete:
AST.get_or_create().delete_unit(unit_type, unit_name)
self._units_to_delete = []
for (unit_type, unit_name, variation_name) in self._var_to_delete:
AST.get_or_create().delete_unit(unit_type, unit_name,
variation_name)
self._var_to_delete = []
def test_variations(capsys):
cmd = HideCommand('hide alias "var#one"')
facade = new_facade()
cmd.execute()
try:
unit = AST.get_or_create()[UnitType.alias]["var"]
assert "one" not in unit._variation_rules
except KeyError:
pytest.fail(
"Unexpected KeyError. Alias 'var' doesn't exist in the parser.")
cmd = UnhideCommand('unhide alias "var#nothing"')
cmd.execute()
captured = capsys.readouterr()
assert "Variation 'nothing' of alias 'var' " + \
"was not previously hidden." in captured.out
cmd = UnhideCommand('unhide ~ "var#one"')
cmd.execute()
try:
unit = AST.get_or_create()[UnitType.alias]["var"]
assert "one" in unit._variation_rules
except KeyError:
pytest.fail("Unexpected KeyError. Alias 'var' doesn't exist " + \
"in the parser.")
new_facade()
cmd = HideCommand('hide ~ "var#nothing"')
cmd.execute()
captured = capsys.readouterr()
assert "Couldn't find variation 'nothing' in alias 'var'." in captured.out
cmd = UnhideCommand('unhide ~ "nothing#var"')
cmd.execute()
captured = capsys.readouterr()
assert "Alias 'nothing' is not defined." in captured.out
# Hide a variation and try to restore it twice
cmd = HideCommand('hide ~ "var#one"')
facade = new_facade()
cmd.execute()
cmd = HideCommand('hide ~ "var#one"')
other_facade = new_facade()
cmd.execute()
cmd = UnhideCommand('unhide ~ "var#one"')
assert cmd.command_tokens == ["unhide", "~", '"var#one"']
cmd.execute()
cmd.execute()
captured = capsys.readouterr()
assert "[ERROR]\tVariation 'one' is " + \
"already defined for alias 'var'." in captured.out
def execute_on_unit(self, unit_type, unit_name, variation_name=None):
if variation_name is None:
try:
unit = HideCommand.stored_units[unit_type.name][unit_name]
AST.get_or_create().add_unit(unit, unit_type)
del HideCommand.stored_units[unit_type.name][unit_name]
self.print_wrapper.write(
unit_type.name.capitalize() + " '" + unit_name + \
"' was successfully restored."
)
except KeyError:
self.print_wrapper.error_log(
unit_type.name.capitalize() + " '" + unit_name + \
"' was not previously hidden."
)
except ValueError:
self.print_wrapper.error_log(
unit_type.name.capitalize() + " '" + unit_name + \
"' is already defined in the parser."
)
else:
unit = None
try:
unit = AST.get_or_create()[unit_type][unit_name]
except KeyError:
self.print_wrapper.error_log(
unit_type.name.capitalize() + " '" + unit_name + \
"' is not defined."
)
return
try:
rules = \
HideCommand.stored_variations[unit_type.name][unit_name][variation_name]
if variation_name in unit._variation_rules:
self.print_wrapper.error_log(
"Variation '" + variation_name + \
"' is already defined for " + unit_type.name + \
" '" + unit_name + "'."
)
return
unit.add_all_rules(rules, variation_name)
self.print_wrapper.write(
"Variation '" + variation_name + "' of " + \
unit_type.name + " '" + unit_name + \
"' was successfully restored."
)
except KeyError:
self.print_wrapper.error_log(
"Variation '" + variation_name + \
"' of " + unit_type.name + " '" + unit_name + \
"' was not previously hidden."
)
def test_execute():
cmd = RenameCommand('rename alias "can you" "could you"')
assert cmd.command_tokens == [
"rename", "alias", '"can you"', '"could you"'
]
facade = new_facade()
cmd.execute()
with pytest.raises(KeyError):
AST.get_or_create()[UnitType.alias]["can you"]
try:
AST.get_or_create()[UnitType.alias]["could you"]
except KeyError:
pytest.fail(
"Unexpected KeyError exception. Renaming didn't properly work.")
cmd = RenameCommand('rename ~ "tell me" "a"')
assert cmd.command_tokens == ["rename", "~", '"tell me"', '"a"']
facade = new_facade()
cmd.execute()
with pytest.raises(KeyError):
AST.get_or_create()[UnitType.alias]["tell me"]
try:
AST.get_or_create()[UnitType.alias]["a"]
except KeyError:
pytest.fail(
"Unexpected KeyError exception. Renaming didn't properly work.")
def finish_execution(self):
for (unit_type, unit_name, variation_name) in self._units_to_delete:
try:
AST.get_or_create().delete_unit(unit_type, unit_name,
variation_name)
self.print_wrapper.write(
unit_type.name.capitalize() + " '" + unit_name + \
"' was successfully deleted."
)
except KeyError:
self.print_wrapper.write(
unit_type.name.capitalize() + " '" + unit_name + \
"' was not defined."
)
self._units_to_delete = []
def create_concrete(self):
self._check_information()
if self.variation is not None:
definitions = AST.get_or_create()[UnitType.slot]
if self.identifier in definitions:
return definitions[self.identifier]
return SlotDefinition(self.identifier, self._build_modifiers_repr())
def execute_on_unit(self, unit_type, unit_name, variation_name=None):
if self.nb_examples is None: # Caching the parsed number
self.nb_examples = -1 # All examples should be generated
if len(self.command_tokens) > 3:
try:
self.nb_examples = int(self.command_tokens[3])
except ValueError:
self.print_wrapper.error_log(
"The number of examples to be generated is invalid: " + \
"it must be an integer (no other characters allowed)."
)
return
try:
definition = \
AST.get_or_create()[unit_type][unit_name]
except KeyError:
self.print_wrapper.write(
unit_type.name.capitalize() + " '" + unit_name + \
"' is not defined."
)
return
if self.nb_examples != -1:
examples = definition.generate_nb_possibilities(
self.nb_examples, variation_name=variation_name
)
else:
examples = definition.generate_all(variation_name=variation_name)
self.print_wrapper.write(
"Examples for " + unit_type.name + " '" + unit_name + "':"
)
for ex in examples:
self.print_wrapper.write(ex)
self.print_wrapper.write("")
def execute_on_unit(self, unit_type, unit_name, variation_name=None):
try:
unit = AST.get_or_create()[unit_type][unit_name]
if variation_name is None:
self.stored_units[unit_type.name][unit_name] = unit
self._units_to_delete.append((unit_type, unit_name))
self.print_wrapper.write(
unit_type.name.capitalize() + " '" + unit_name + \
"' was successfully hidden."
)
else:
if variation_name not in unit._variation_rules:
self.print_wrapper.error_log(
"Couldn't find variation '" + variation_name + \
"' in " + unit_type.name + " '" + unit_name + "'."
)
return
self._var_to_delete.append((unit_type, unit_name, variation_name))
rules = unit._variation_rules[variation_name]
if unit_name not in self.stored_variations[unit_type.name]:
self.stored_variations[unit_type.name][unit_name] = \
{variation_name: rules}
else:
self.stored_variations[unit_type.name][unit_name][variation_name] = \
rules
self.print_wrapper.write(
"Variation '" + variation_name + "' of " + \
unit_type.name + " '" + unit_name + \
"' was successfully hidden."
)
except KeyError:
self.print_wrapper.write(
unit_type.name.capitalize() + " '" + unit_name + \
"' was not defined."
)
def run_generation(self, adapter_str=None):
""""
Runs the generation of all intents and writes them out to the output
file(s) using the adapter `adapter` if one is provided.
@pre: the parsing has been done.
"""
if adapter_str is None:
adapter = self.adapter
else:
adapter = adapter_factory.create_adapter(adapter_str)
self.generator = Generator()
synonyms = AST.get_or_create().get_entities_synonyms()
if os.path.exists(self.output_dir_path):
if self.force_overwriting or self._ask_confirmation():
shutil.rmtree(self.output_dir_path)
else:
print_DBG("Aborting generation. Exiting without any change.")
return
train_examples = list(self.generator.generate_train())
if train_examples:
adapter.write(os.path.join(self.output_dir_path, "train"),
train_examples, synonyms)
test_examples = list(self.generator.generate_test(train_examples))
if test_examples:
adapter.write(os.path.join(self.output_dir_path, "test"),
test_examples, synonyms)
print_DBG("Generation over")
def __init__(self):
self.ast = AST.get_or_create()
total_nb_units = len(self.ast[UnitType.intent]) + len(
self.ast[UnitType.slot]) + len(self.ast[UnitType.alias])
if total_nb_units >= 50:
Configuration.get_or_create().set_caching_level(0)
def execute(self):
"""
Implements the command `rule` which generates a certain number of
examples according to a provided rule.
"""
if len(self.command_tokens) < 3:
self.print_wrapper.error_log("Missing some arguments\nUsage: " +
self.usage_str)
return
unit_type = \
CommandStrategy.get_unit_type_from_str(self.command_tokens[1])
if unit_type is None:
self.print_wrapper.error_log("Unknown unit type: '" +
str(self.command_tokens[1]) + "'.")
return
try:
[unit_name, variation_name] = \
CommandStrategy.split_exact_unit_name(self.command_tokens[2])
except SyntaxError:
self.print_wrapper.error_log(
"Unit identifier couldn't be interpreted. " + \
"Did you mean to escape some hashtags '#'?"
)
return
if variation_name is not None and variation_name != "":
self.print_wrapper.error_log(
"Variation name detected, while units cannot be " + \
"declared with a variation. " + \
"Did you mean to escape some hashtags '#'?"
)
return
relevant_dict = AST.get_or_create()[unit_type]
if unit_type == UnitType.alias:
builder = AliasDefBuilder()
builder.identifier = unit_name
declaration = builder.create_concrete()
elif unit_type == UnitType.slot:
builder = SlotDefBuilder()
builder.identifier = unit_name
declaration = builder.create_concrete()
else: # intent
builder = IntentDefBuilder()
builder.identifier = unit_name
declaration = builder.create_concrete()
if unit_name in relevant_dict:
self.print_wrapper.write(
unit_type.name.capitalize() + " '" + unit_name + \
"' was NOT defined, as it already is defined."
)
return
relevant_dict[unit_name] = declaration
self.print_wrapper.write(
unit_type.name.capitalize() + " '" + unit_name + \
"' was successfully declared."
)
def create_concrete(self):
self._check_information()
if self.variation is not None:
definitions = AST.get_or_create()[UnitType.intent]
if self.identifier in definitions:
return definitions[self.identifier]
return IntentDefinition(self.identifier, self._build_modifiers_repr(),
self.nb_training_ex, self.nb_testing_ex)
def test_generate_all_training(self):
"""
Tests templates that generate all possible examples for each intent
and that generate only training data.
"""
facade = ChatetteFacade.get_or_create()
input_dir_path = "tests/system-testing/inputs/generate-all/"
input_filenames = [
"simplest.chatette", "only-words.chatette",
"words-and-groups.chatette", "alias.chatette", "include.chatette",
"slot.chatette"
]
for filename in input_filenames:
file_path = os.path.join(input_dir_path, filename)
facade.run(file_path)
if not TestSystem.check_no_duplicates(facade.train_examples):
pytest.fail(
"Some examples were generated several times " +
"when dealing with file '" + filename + "'.\nGenerated: " + \
str(facade.train_examples)
)
legal_examples = TestSystem.get_legal_examples(file_path)
for ex in facade.train_examples:
formatted_ex = {"intent": ex.intent_name, "text": ex.text}
if formatted_ex not in legal_examples:
pytest.fail(
str(formatted_ex) + " is not a legal example for '" + \
file_path + "'"
)
if len(legal_examples) != len(facade.train_examples):
training_texts = [ex.text for ex in facade.train_examples]
for legal_ex in legal_examples:
if legal_ex["text"] not in training_texts:
pytest.fail(
"Example '" + legal_ex["text"] + \
"' was not generated."
)
pytest.fail(
"An unknown example was not generated (" + \
str(len(facade.train_examples)) + \
" generated instead of " + str(len(legal_examples)) + \
").\nGenerated: " + str(facade.train_examples)
)
legal_syn = TestSystem.get_legal_synonyms(file_path)
if legal_syn is not None:
synonyms = AST.get_or_create().get_entities_synonyms()
for key in synonyms:
if key not in legal_syn:
pytest.fail(
"'" + key + "' shouldn't have any synonyms."
)
for syn in synonyms[key]:
if syn not in legal_syn[key]:
pytest.fail(
"'" + syn + "' shouldn't be a synonym of '" + \
key + "'"
)
def __init__(self, identifier, unit_type, leading_space, modifiers):
self._unit_type = unit_type
super(UnitReference, self).__init__(identifier, leading_space,
modifiers)
try:
self._definition = AST.get_or_create()[unit_type][identifier]
self._fix_arg_value_modifier()
except KeyError:
self._definition = None
def test_execute(capsys):
print("AAAAAAAAAAAA")
facade = new_facade()
cmd = DeclareCommand('declare alias "machin"')
cmd.execute()
try:
unit = AST.get_or_create()[UnitType.alias]["machin"]
assert len(unit._variation_rules) == 0
assert len(unit._all_rules) == 0
except (KeyError, ValueError):
pytest.fail("Unexpected error, 'declare' command didn't work.")
captured = capsys.readouterr()
assert "Alias 'machin' was successfully declared." in captured.out
print("AAAAAAAAAAAA")
facade = new_facade()
cmd = DeclareCommand('declare slot "machin"')
cmd.execute()
try:
unit = AST.get_or_create()[UnitType.slot]["machin"]
assert len(unit._variation_rules) == 0
assert len(unit._all_rules) == 0
except (KeyError, ValueError):
pytest.fail("Unexpected error, 'declare' command didn't work.")
captured = capsys.readouterr()
assert "Slot 'machin' was successfully declared." in captured.out
print("AAAAAAAAAAAA")
facade = new_facade()
cmd = DeclareCommand('declare intent "machin"')
cmd.execute()
try:
unit = AST.get_or_create()[UnitType.intent]["machin"]
assert len(unit._variation_rules) == 0
assert len(unit._all_rules) == 0
except (KeyError, ValueError):
pytest.fail("Unexpected error, 'declare' command didn't work.")
captured = capsys.readouterr()
assert "Intent 'machin' was successfully declared." in captured.out
def test_execute(capsys):
new_facade()
cmd = DeleteCommand('delete alias "inexistant"')
cmd.execute()
captured = capsys.readouterr()
assert "Alias 'inexistant' was not defined." in captured.out
cmd = DeleteCommand('delete alias "tell me"')
facade = new_facade()
try:
AST.get_or_create()[UnitType.alias]["tell me"]
except KeyError:
pytest.fail("Unexpected KeyError. Alias 'tell me' doesn't exist " + \
"in the parser.")
cmd.execute()
with pytest.raises(KeyError):
AST.get_or_create()[UnitType.alias]["tell me"]
captured = capsys.readouterr()
assert "Alias 'tell me' was successfully deleted." in captured.out
def get_all_matching_unit_names(self, unit_type, regex):
"""
Returns a list of unit names of type `unit_type`
whose name matches `regex`.
NOTE: this is used with 'delete' command since the generator that
returns the same data cannot be used in that case (dict changes
size during iteration).
"""
relevant_dict = AST.get_or_create()[unit_type]
if self._is_regex_global:
return [name for name in relevant_dict if regex.search(name)]
return [name for name in relevant_dict if regex.match(name)]
def test_new_variation(self):
builder = AliasDefBuilder()
builder.identifier = "id"
alias = builder.create_concrete()
assert "var" not in alias
ast = AST.get_or_create()
ast.add_alias(alias)
builder.variation = "var"
same_alias = builder.create_concrete()
assert same_alias == alias
def get_definition(self):
if self._definition is None:
try:
self._definition = \
AST.get_or_create()[self._unit_type][self._name]
self._fix_arg_value_modifier()
except KeyError:
raise KeyError(
"Couldn't find the definition corresponding to " + \
self.full_name + "."
)
return self._definition
def _add_rule(self, unit_type, unit_name, variation_name, rule_str):
parser = Facade.get_or_create().parser
rule_tokens = parser.lexer.lex("\t" + rule_str)
rule = parser._parse_rule(rule_tokens[1:])
unit = AST.get_or_create()[unit_type][unit_name]
unit.add_rule(rule, variation_name)
self.print_wrapper.write(
"Rule successfully added to " + unit_type.name + " '" + \
unit_name + "'."
)
def test_new_variation(self):
builder = IntentDefBuilder()
builder.identifier = "id"
intent = builder.create_concrete()
assert "var" not in intent
ast = AST.get_or_create()
ast.add_intent(intent)
builder.variation = "var"
same_intent = builder.create_concrete()
assert same_intent == intent
def test_new_variation(self):
builder = SlotDefBuilder()
builder.identifier = "id"
slot = builder.create_concrete()
assert "var" not in slot
ast = AST.get_or_create()
ast.add_slot(slot)
builder.variation = "var"
same_slot = builder.create_concrete()
assert same_slot == slot
def execute(self):
"""
Implements the command `rename` which renames a unit
into something else. Displays an error if the unit wasn't found.
"""
if len(self.command_tokens) < 4:
self.print_wrapper.error_log(
"Missing some arguments\nUsage: " + \
'rename <unit-type> "<old-name>" ' + '"<new-name>"')
return
unit_type = \
CommandStrategy.get_unit_type_from_str(self.command_tokens[1])
if unit_type is None:
self.print_wrapper.error_log(
"Unknown unit type: '" + str(self.command_tokens[1]) + "'."
)
else:
old_name = CommandStrategy.remove_quotes(self.command_tokens[2])
new_name = CommandStrategy.remove_quotes(self.command_tokens[3])
if new_name == "":
self.print_wrapper.error_log(
"An empty name is not a valid " + unit_type.name + " name."
)
return
try:
AST.get_or_create().rename_unit(unit_type, old_name, new_name)
self.print_wrapper.write(
unit_type.name.capitalize() + " '" + old_name + \
"' was successfully renamed to '" + new_name + "'.")
except KeyError:
self.print_wrapper.error_log(
"Couldn't find a unit named '" + str(old_name) + "'."
)
except ValueError:
self.print_wrapper.error_log(
unit_type.name.capitalize() + " '" + new_name + \
"' is already in use."
)
def next_matching_unit_name(self, unit_type, regex):
"""
Yields the next unit name of type `unit_type`
whose name matches `regex`.
"""
relevant_dict = AST.get_or_create()._get_relevant_dict(unit_type)
if self._is_regex_global:
for unit_name in relevant_dict:
if regex.search(unit_name):
yield unit_name
else:
for unit_name in relevant_dict:
if regex.match(unit_name):
yield unit_name
def execute_on_unit(self, unit_type, unit_name, variation_name=None):
try:
unit = AST.get_or_create()[unit_type][unit_name]
self.print_wrapper.write(unit.short_description())
if variation_name is None:
self.print_wrapper.write("Template rules:\n" +
str(unit.as_template_str()))
else: # TODO
pass
except KeyError:
self.print_wrapper.write(
unit_type.name.capitalize() + " '" + unit_name + \
"' is not defined."
)
def _set_modifier(self, unit_type, unit_name, modifier_name, value):
unit = AST.get_or_create()[unit_type][unit_name]
modifier_name = modifier_name.lower()
if modifier_name in ("casegen", CASE_GEN_SYM):
try:
value = str_to_bool(value)
unit.set_casegen(value)
except ValueError:
self.print_wrapper.write(
"Invalid value for case generation modifier " + \
"(True or False)."
)
return
elif modifier_name in ("arg", ARG_SYM):
unit.set_arg_value(value)
elif modifier_name in ("randgen", RAND_GEN_SYM):
try:
value = str_to_bool(value)
unit.set_randgen(value)
except ValueError:
self.print_wrapper.write(
"Invalid value for random generation modifier " + \
"(True or False)."
)
return
elif modifier_name in "randgen-name":
unit.set_randgen_name(modifier_name)
elif modifier_name in ("randgen-percent", RAND_GEN_PERCENT_SYM):
try:
value = float(value)
unit.set_randgen_percent(value)
except ValueError:
self.print_wrapper.write(
"Invalid value for the random generation percentage " + \
"modifier (expected a float)."
)
return
else:
self.print_wrapper.write(
"Invalid modifier selected (can be 'casegen', " + \
"'randgen', 'randgen-name', 'randgen-percent' or 'arg')."
)
return
self.print_wrapper.write(
"Modifier for " + unit_type.name + " '" + unit_name + \
"' successfully changed."
)
def __init__(self, master_file_path):
if (master_file_path is not None
and not isinstance(master_file_path, string_types)):
raise ValueError(
"Since v1.4.0, the parser takes as an argument " + \
"the path of the master file directly, rather " + \
"than the file itself as before.")
self._master_filepath = master_file_path
self.input_file_manager = \
InputFileManager.get_or_create(master_file_path)
self.lexer = Lexer()
self.ast = AST.get_or_create()
self._declaration_line_allowed = True
self._last_indentation = None
self._current_unit_declaration = None
self._current_variation_name = None
def write(self, adapter="rasa"):
if self.generator is None:
raise ValueError("Tried to write an output file before generation")
if adapter == "rasa":
adapter = self.rasa_adapter
elif adapter == "jsonl":
adapter = self.jsonl_adapter
else:
raise ValueError(adapter + " is not a valid adapter.")
synonyms = AST.get_or_create().get_entities_synonyms()
if self.train_examples:
adapter.write(os.path.join(self.output_dirpath, "train"),
self.train_examples, synonyms)
if self.test_examples:
adapter.write(os.path.join(self.output_dirpath, "train"),
self.test_examples, synonyms)
def execute_on_unit(self, unit_type, unit_name, variation_name=None):
try:
unit = AST.get_or_create()[unit_type][unit_name]
self.print_wrapper.write(unit.short_description())
if variation_name is not None:
if variation_name in unit._variation_rules:
self.print_wrapper.write(
"Variation '" + variation_name + \
"' is defined for this " + unit.unit_type.name + "."
)
else:
self.print_wrapper.write(
"Variation '" + variation_name + \
"' is not defined for this " + unit.unit_type.name + "."
)
except KeyError:
self.print_wrapper.write(
unit_type.name.capitalize() + " '" + unit_name + \
"' is not defined."
)
self.print_wrapper.write("")