def test_nb(self): """Tests that the cast doesn't do anything for numeric types.""" res_int = cast_to_unicode(5) assert res_int == 5 res_float = cast_to_unicode(3.14159265) assert res_float == 3.14159265 res_complex = cast_to_unicode(complex(1,2)) assert res_complex == complex(1,2)
def warn_old_comment(self, filename=None, line_nb=None, line=None): """ Warns the user on stderr that one of their files contains semicolons comments (which are a deprecated way of making comments). Rather use '//' comments instead of ';' comments. """ if not self._old_comment_warned: self._old_comment_warned = True message = \ "Comments starting with a semi-colon '" + \ OLD_COMMENT_SYM + "' are now deprecated. " + \ "Please use the new double slash '" + COMMENT_SYM + \ "' syntax instead." if filename is not None: message += \ "\nThis syntax was found in file '" + \ cast_to_unicode(filename) + "'" if line_nb is not None and line is not None: message += \ " at line " + str(line_nb) + ": '" + \ str(line).strip() + "'" message += '.' elif line_nb is not None and line is not None: message += \ "\nThis syntax was found at line " + str(line_nb) + \ ": '" + str(line).strip() + "'." warn(message, DeprecationWarning) print_warn(message)
def warn_old_choice(self, filename=None, line_nb=None, line=None): """ Warns the user on stderr that one of their files contains semicolons comments (which are a deprecated way of making comments). Rather use '//' comments instead of ';' comments. """ if not self._old_choice_warned: self._old_choice_warned = True message = \ "Choices starting with '" + OLD_CHOICE_START + \ "' and ending with '" + OLD_CHOICE_END + \ "' are now deprecated. Please use the new syntax that " + \ "starts with '" + CHOICE_START + "' and ends with '" + \ CHOICE_END + "' instead." if filename is not None: message += \ "\nThis syntax was found in file '" + \ cast_to_unicode(filename) + "'" if line_nb is not None and line is not None: message += \ " at line " + str(line_nb) + ": '" + \ str(line).strip() + "'" message += '.' elif line_nb is not None and line is not None: message += \ "\nThis syntax was found at line " + str(line_nb) + \ ": '" + str(line).strip() + "'." warn(message, DeprecationWarning) print_warn(message)
def test_dict(self): dicts = [{"a": "b"}, {}, {"c": 0, 1: "d"}, {"e": {"f": "g", 0: ["h", 3]}}] for d in dicts: res_dict = cast_to_unicode(d) if sys.version_info[0] == 3: assert res_dict == d else: self.check_is_unicode(res_dict)
def _write_batch(self, output_file_handle, batch): prepared_examples = dict() for example in batch.examples: append_to_list_in_dict(prepared_examples, example.intent_name, self.prepare_example(example)) for intent_name in prepared_examples: output_file_handle.write("## intent:" + cast_to_unicode(intent_name) + '\n') for text in prepared_examples[intent_name]: output_file_handle.write(cast_to_unicode("- " + text + '\n')) output_file_handle.write(cast_to_unicode('\n')) output_file_handle.write( cast_to_unicode(self.__format_synonyms(batch.synonyms))) remainder = self._get_base_to_extend() if remainder is not None: output_file_handle.write(cast_to_unicode(remainder) + '\n')
def write(self, output_directory, examples, synonyms): super(JsonListAdapter, self).write(output_directory, examples, synonyms) processed_synonyms = self.__synonym_format(synonyms) if processed_synonyms is not None: synonyms_file_path = os.path.join(output_directory, "synonyms.json") with io.open(synonyms_file_path, 'w') as output_file: output_file.write(json.dumps(cast_to_unicode(processed_synonyms), ensure_ascii=False, sort_keys=True, indent=2))
def write(self, output_directory, examples, synonyms): #def write(self, output_directory, examples: List[IntentExample], synonyms) -> None: super(JsonListAdapter, self).write(output_directory, examples, synonyms) synonyms_file_path = os.path.join(output_directory, "synonyms.json") with io.open(synonyms_file_path, 'w', encoding="utf-8") as output_file: output_file.write( json.dumps(cast_to_unicode(synonyms), ensure_ascii=False, sort_keys=True, indent=2))
def _write_batch(self, output_file_handle, batch): rasa_entities = [self.prepare_example(ex) for ex in batch.examples] json_data = self._get_base_to_extend() json_data["rasa_nlu_data"]["common_examples"] = rasa_entities json_data["rasa_nlu_data"]["entity_synonyms"] = \ self.__format_synonyms(batch.synonyms) json_data = cast_to_unicode(json_data) output_file_handle.write( json.dumps(json_data, ensure_ascii=False, indent=2, sort_keys=True))
def open_file(self, file_path): """ Opens the file at `file_path` if and only if it wasn't opened before. Stores the currently read file for further reading if needed. `file_path` is understood with respect to the currently being parsed file (rather than working directory), unless it is an absolute path or there is no file currently being parsed. @raises: - `FileAlreadyOpened` if the file at `file_path` was already opened. """ file_path = cast_to_unicode(file_path) if not os.path.isabs(file_path): if self._current_file is None: file_path = cast_to_unicode(os.path.abspath(file_path)) else: file_path = \ os.path.join( cast_to_unicode(os.path.dirname(self._current_file.name)), file_path ) opened_file_paths = [f.name for f in self._opened_files] if file_path in opened_file_paths: raise FileAlreadyOpened("Tried to read file '" + file_path + "' several times.") if self._current_file is not None: self._opened_files.append(self._current_file) try: self._current_file = LineCountFileWrapper(file_path) Stats.get_or_create().new_file_parsed() except IOError as e: if len(self._opened_files) > 0: self._current_file = self._opened_files.pop() raise e
def _write_batch(self, output_file_handle, batch): rasa_entities = [self.prepare_example(ex) for ex in batch.examples] json_data = { "rasa_nlu_data": { "common_examples": rasa_entities, "regex_features": [], "entity_synonyms": self.__synonym_format(batch.synonyms), } } json_data = cast_to_unicode(json_data) output_file_handle.write( json.dumps(json_data, ensure_ascii=False, indent=2, sort_keys=True))
def open_new_file(self, filepath): """Opens the new (master) file, making the parser ready to parse it.""" try: self.input_file_manager.open_file(filepath) except IOError as e: raise IOError( "There was an error while opening file '" + \ str(cast_to_unicode(filepath)) + "': " + str(e) + "." ) except FileAlreadyOpened as e: err_msg = str(e) current_file_name = self.input_file_manager.get_current_file_name() if current_file_name is not None: err_msg += \ "\nContinuing the parsing of '" + str(current_file_name) + \ "'." print_warn(err_msg)
def _write_batch(self, output_file_handle, batch): #def _write_batch(self, output_file_handle: TextIO, batch: Batch) -> None: def example_to_rasa_entities(example): #def example_to_rasa_entities(example: IntentExample): def entity_to_rasa(entity): entity["text"] = entity["text"].strip() first_index = self.__find_entity(example.text, entity["text"]) # NOTE: This always finds something # Remove the entity marker of this entity # (works unless entities are not recorded in order) example.text = example.text[:first_index] + \ example.text[first_index+len(ENTITY_MARKER):] return { "value": entity["value"], "entity": entity["slot-name"], "start": first_index, "end": first_index + len(entity["text"]), } return { "intent": example.name, "entities": [entity_to_rasa(e) for e in example.entities], # HACK: Keep "text" after having called `entity_to_rasa` # (removes the entity markers) "text": example.text, } rasa_entities = [example_to_rasa_entities(ex) for ex in batch.examples] json_data = { "rasa_nlu_data": { "common_examples": rasa_entities, "regex_features": [], "entity_synonyms": self.__synonym_format(batch.synonyms), } } json_data = cast_to_unicode(json_data) output_file_handle.write( json.dumps(json_data, ensure_ascii=False, indent=2, sort_keys=True))
def execute(self): if len(self.command_tokens) < 2: self.print_wrapper.error_log("Missing some arguments\nUsage: " + self.usage_str) return template_filepath = self.command_tokens[1] definitions = AST.get_or_create() with io.open(template_filepath, 'w+') as f: for intent_name in definitions._intent_definitions: intent = definitions._intent_definitions[intent_name] print(cast_to_unicode(intent.as_template_str() + '\n'), file=f) print(cast_to_unicode(''), file=f) for alias_name in definitions._alias_definitions: alias = definitions._alias_definitions[alias_name] print(cast_to_unicode(alias.as_template_str() + '\n'), file=f) print(cast_to_unicode(''), file=f) for slot_name in definitions._slot_definitions: slot = definitions._slot_definitions[slot_name] print(cast_to_unicode(slot.as_template_str() + '\n'), file=f) print(cast_to_unicode(''), file=f) self.print_wrapper.write("Template file successfully written.")
def prepare_example(self, example): example.text = example.text.replace(ENTITY_MARKER, "") return json.dumps(cast_to_unicode(example.__dict__), ensure_ascii=False, sort_keys=True)
def prepare_example(self, example): return json.dumps(cast_to_unicode(example.as_dict()), ensure_ascii=False, sort_keys=True)
def __init__(self, filepath, mode='r'): self.name = cast_to_unicode(filepath) self.f = io.open(filepath, mode, encoding='utf-8') self.line_nb = 0