def test_nb(self):
"""Tests that the cast doesn't do anything for numeric types."""
res_int = cast_to_unicode(5)
assert res_int == 5
res_float = cast_to_unicode(3.14159265)
assert res_float == 3.14159265
res_complex = cast_to_unicode(complex(1,2))
assert res_complex == complex(1,2)
def warn_old_comment(self, filename=None, line_nb=None, line=None):
"""
Warns the user on stderr that one of their files contains semicolons
comments (which are a deprecated way of making comments).
Rather use '//' comments instead of ';' comments.
"""
if not self._old_comment_warned:
self._old_comment_warned = True
message = \
"Comments starting with a semi-colon '" + \
OLD_COMMENT_SYM + "' are now deprecated. " + \
"Please use the new double slash '" + COMMENT_SYM + \
"' syntax instead."
if filename is not None:
message += \
"\nThis syntax was found in file '" + \
cast_to_unicode(filename) + "'"
if line_nb is not None and line is not None:
message += \
" at line " + str(line_nb) + ": '" + \
str(line).strip() + "'"
message += '.'
elif line_nb is not None and line is not None:
message += \
"\nThis syntax was found at line " + str(line_nb) + \
": '" + str(line).strip() + "'."
warn(message, DeprecationWarning)
print_warn(message)
def warn_old_choice(self, filename=None, line_nb=None, line=None):
"""
Warns the user on stderr that one of their files contains semicolons
comments (which are a deprecated way of making comments).
Rather use '//' comments instead of ';' comments.
"""
if not self._old_choice_warned:
self._old_choice_warned = True
message = \
"Choices starting with '" + OLD_CHOICE_START + \
"' and ending with '" + OLD_CHOICE_END + \
"' are now deprecated. Please use the new syntax that " + \
"starts with '" + CHOICE_START + "' and ends with '" + \
CHOICE_END + "' instead."
if filename is not None:
message += \
"\nThis syntax was found in file '" + \
cast_to_unicode(filename) + "'"
if line_nb is not None and line is not None:
message += \
" at line " + str(line_nb) + ": '" + \
str(line).strip() + "'"
message += '.'
elif line_nb is not None and line is not None:
message += \
"\nThis syntax was found at line " + str(line_nb) + \
": '" + str(line).strip() + "'."
warn(message, DeprecationWarning)
print_warn(message)
def test_dict(self):
dicts = [{"a": "b"}, {}, {"c": 0, 1: "d"}, {"e": {"f": "g", 0: ["h", 3]}}]
for d in dicts:
res_dict = cast_to_unicode(d)
if sys.version_info[0] == 3:
assert res_dict == d
else:
self.check_is_unicode(res_dict)
def _write_batch(self, output_file_handle, batch):
prepared_examples = dict()
for example in batch.examples:
append_to_list_in_dict(prepared_examples, example.intent_name,
self.prepare_example(example))
for intent_name in prepared_examples:
output_file_handle.write("## intent:" +
cast_to_unicode(intent_name) + '\n')
for text in prepared_examples[intent_name]:
output_file_handle.write(cast_to_unicode("- " + text + '\n'))
output_file_handle.write(cast_to_unicode('\n'))
output_file_handle.write(
cast_to_unicode(self.__format_synonyms(batch.synonyms)))
remainder = self._get_base_to_extend()
if remainder is not None:
output_file_handle.write(cast_to_unicode(remainder) + '\n')
def write(self, output_directory, examples, synonyms):
super(JsonListAdapter, self).write(output_directory, examples, synonyms)
processed_synonyms = self.__synonym_format(synonyms)
if processed_synonyms is not None:
synonyms_file_path = os.path.join(output_directory, "synonyms.json")
with io.open(synonyms_file_path, 'w') as output_file:
output_file.write(json.dumps(cast_to_unicode(processed_synonyms),
ensure_ascii=False,
sort_keys=True, indent=2))
def write(self, output_directory, examples, synonyms):
#def write(self, output_directory, examples: List[IntentExample], synonyms) -> None:
super(JsonListAdapter, self).write(output_directory, examples,
synonyms)
synonyms_file_path = os.path.join(output_directory, "synonyms.json")
with io.open(synonyms_file_path, 'w', encoding="utf-8") as output_file:
output_file.write(
json.dumps(cast_to_unicode(synonyms),
ensure_ascii=False,
sort_keys=True,
indent=2))
def _write_batch(self, output_file_handle, batch):
rasa_entities = [self.prepare_example(ex) for ex in batch.examples]
json_data = self._get_base_to_extend()
json_data["rasa_nlu_data"]["common_examples"] = rasa_entities
json_data["rasa_nlu_data"]["entity_synonyms"] = \
self.__format_synonyms(batch.synonyms)
json_data = cast_to_unicode(json_data)
output_file_handle.write(
json.dumps(json_data, ensure_ascii=False, indent=2,
sort_keys=True))
def open_file(self, file_path):
"""
Opens the file at `file_path` if and only if it wasn't opened before.
Stores the currently read file for further reading if needed.
`file_path` is understood with respect to the currently being parsed
file (rather than working directory),
unless it is an absolute path or there is no file currently being
parsed.
@raises: - `FileAlreadyOpened` if the file at `file_path`
was already opened.
"""
file_path = cast_to_unicode(file_path)
if not os.path.isabs(file_path):
if self._current_file is None:
file_path = cast_to_unicode(os.path.abspath(file_path))
else:
file_path = \
os.path.join(
cast_to_unicode(os.path.dirname(self._current_file.name)),
file_path
)
opened_file_paths = [f.name for f in self._opened_files]
if file_path in opened_file_paths:
raise FileAlreadyOpened("Tried to read file '" + file_path +
"' several times.")
if self._current_file is not None:
self._opened_files.append(self._current_file)
try:
self._current_file = LineCountFileWrapper(file_path)
Stats.get_or_create().new_file_parsed()
except IOError as e:
if len(self._opened_files) > 0:
self._current_file = self._opened_files.pop()
raise e
def _write_batch(self, output_file_handle, batch):
rasa_entities = [self.prepare_example(ex) for ex in batch.examples]
json_data = {
"rasa_nlu_data": {
"common_examples": rasa_entities,
"regex_features": [],
"entity_synonyms": self.__synonym_format(batch.synonyms),
}
}
json_data = cast_to_unicode(json_data)
output_file_handle.write(
json.dumps(json_data, ensure_ascii=False, indent=2,
sort_keys=True))
def open_new_file(self, filepath):
"""Opens the new (master) file, making the parser ready to parse it."""
try:
self.input_file_manager.open_file(filepath)
except IOError as e:
raise IOError(
"There was an error while opening file '" + \
str(cast_to_unicode(filepath)) + "': " + str(e) + "."
)
except FileAlreadyOpened as e:
err_msg = str(e)
current_file_name = self.input_file_manager.get_current_file_name()
if current_file_name is not None:
err_msg += \
"\nContinuing the parsing of '" + str(current_file_name) + \
"'."
print_warn(err_msg)
def _write_batch(self, output_file_handle, batch):
#def _write_batch(self, output_file_handle: TextIO, batch: Batch) -> None:
def example_to_rasa_entities(example):
#def example_to_rasa_entities(example: IntentExample):
def entity_to_rasa(entity):
entity["text"] = entity["text"].strip()
first_index = self.__find_entity(example.text, entity["text"])
# NOTE: This always finds something
# Remove the entity marker of this entity
# (works unless entities are not recorded in order)
example.text = example.text[:first_index] + \
example.text[first_index+len(ENTITY_MARKER):]
return {
"value": entity["value"],
"entity": entity["slot-name"],
"start": first_index,
"end": first_index + len(entity["text"]),
}
return {
"intent": example.name,
"entities": [entity_to_rasa(e) for e in example.entities],
# HACK: Keep "text" after having called `entity_to_rasa`
# (removes the entity markers)
"text": example.text,
}
rasa_entities = [example_to_rasa_entities(ex) for ex in batch.examples]
json_data = {
"rasa_nlu_data": {
"common_examples": rasa_entities,
"regex_features": [],
"entity_synonyms": self.__synonym_format(batch.synonyms),
}
}
json_data = cast_to_unicode(json_data)
output_file_handle.write(
json.dumps(json_data, ensure_ascii=False, indent=2,
sort_keys=True))
def execute(self):
if len(self.command_tokens) < 2:
self.print_wrapper.error_log("Missing some arguments\nUsage: " +
self.usage_str)
return
template_filepath = self.command_tokens[1]
definitions = AST.get_or_create()
with io.open(template_filepath, 'w+') as f:
for intent_name in definitions._intent_definitions:
intent = definitions._intent_definitions[intent_name]
print(cast_to_unicode(intent.as_template_str() + '\n'), file=f)
print(cast_to_unicode(''), file=f)
for alias_name in definitions._alias_definitions:
alias = definitions._alias_definitions[alias_name]
print(cast_to_unicode(alias.as_template_str() + '\n'), file=f)
print(cast_to_unicode(''), file=f)
for slot_name in definitions._slot_definitions:
slot = definitions._slot_definitions[slot_name]
print(cast_to_unicode(slot.as_template_str() + '\n'), file=f)
print(cast_to_unicode(''), file=f)
self.print_wrapper.write("Template file successfully written.")
def prepare_example(self, example):
example.text = example.text.replace(ENTITY_MARKER, "")
return json.dumps(cast_to_unicode(example.__dict__),
ensure_ascii=False,
sort_keys=True)
def prepare_example(self, example):
return json.dumps(cast_to_unicode(example.as_dict()),
ensure_ascii=False,
sort_keys=True)
def __init__(self, filepath, mode='r'):
self.name = cast_to_unicode(filepath)
self.f = io.open(filepath, mode, encoding='utf-8')
self.line_nb = 0
