def run(self): stream = self.context.stream # If the macro tokenizer was called with an unkown opening delimiter sequence, mark it as an error and exit if self.opening_delimiter != self.__class__.OPENING_DELIMITER: yield Tokens.ERROR(self.opening_delimiter, self.opening_delimiter_position, self.opening_delimiter_position_after, "Comment tokenizer called with unknown opening sequence “%s”" % self.opening_delimiter) return stream.push() seen_escape = False opening_comment_token = Tokens.BEGIN_MACRO(self.__class__.OPENING_DELIMITER, self.opening_delimiter_position, self.opening_delimiter_position_after) yield opening_comment_token value = "" value_first_position = stream.copy_absolute_position() while True: if stream.next_is_EOF(): yield Tokens.COMMENT(value, value_first_position, stream.copy_absolute_position()) yield Tokens.END_MACRO(opening_comment_token, "", stream.copy_absolute_position(), stream.copy_absolute_position()) stream.pop() return char = stream.read() if char == '\\': if seen_escape: value += '\\' else: seen_escape = True else: if seen_escape: if char == self.__class__.CLOSING_DELIMITER: value += self.__class__.CLOSING_DELIMITER elif char == '$': value += '$' else: yield Tokens.COMMENT(value, value_first_position, stream.absolute_position_of_unread()) value = "" value_first_position = stream.copy_absolute_position() yield Tokens.ERROR(char, stream.absolute_position_of_unread(), stream.copy_absolute_position(), "Unknown escape code sequence “%s”." % char) seen_escape = False else: if char == self.__class__.CLOSING_DELIMITER: yield Tokens.COMMENT(value, value_first_position, stream.absolute_position_of_unread()) yield Tokens.END_MACRO(opening_comment_token, self.__class__.CLOSING_DELIMITER, stream.absolute_position_of_unread(), stream.copy_absolute_position()) stream.pop() return elif char == '$': yield Tokens.COMMENT(value, value_first_position, stream.absolute_position_of_unread()) value = "" value_first_position = stream.copy_absolute_position() for token in util.interpolation(self.context): yield token else: value += char
def run(self): stream = self.context.stream # If the macro tokenizer was called with an unkown opening delimiter sequence, mark it as an error and exit if self.opening_delimiter != self.__class__.OPENING_DELIMITER: yield Tokens.ERROR( self.opening_delimiter, self.opening_delimiter_position, self.opening_delimiter_position_after, "Comment tokenizer called with unknown opening sequence “%s”" % self.opening_delimiter) return stream.push() seen_escape = False opening_comment_token = Tokens.BEGIN_MACRO( self.__class__.OPENING_DELIMITER, self.opening_delimiter_position, self.opening_delimiter_position_after) yield opening_comment_token value = "" value_first_position = stream.copy_absolute_position() while True: if stream.next_is_EOF(): yield Tokens.COMMENT(value, value_first_position, stream.copy_absolute_position()) yield Tokens.END_MACRO(opening_comment_token, "", stream.copy_absolute_position(), stream.copy_absolute_position()) stream.pop() return char = stream.read() if char == '\\': if seen_escape: value += '\\' else: seen_escape = True else: if seen_escape: if char == self.__class__.CLOSING_DELIMITER: value += self.__class__.CLOSING_DELIMITER elif char == '$': value += '$' else: yield Tokens.COMMENT( value, value_first_position, stream.absolute_position_of_unread()) value = "" value_first_position = stream.copy_absolute_position() yield Tokens.ERROR( char, stream.absolute_position_of_unread(), stream.copy_absolute_position(), "Unknown escape code sequence “%s”." % char) seen_escape = False else: if char == self.__class__.CLOSING_DELIMITER: yield Tokens.COMMENT( value, value_first_position, stream.absolute_position_of_unread()) yield Tokens.END_MACRO( opening_comment_token, self.__class__.CLOSING_DELIMITER, stream.absolute_position_of_unread(), stream.copy_absolute_position()) stream.pop() return elif char == '$': yield Tokens.COMMENT( value, value_first_position, stream.absolute_position_of_unread()) value = "" value_first_position = stream.copy_absolute_position() for token in util.interpolation(self.context): yield token else: value += char
def run(self): stream = self.context.stream readtable = self.context.readtable if stream.next_is_EOF(): yield Tokens.ERROR(self.__class__.MY_OPENING_DELIMITER, self.opening_delimiter_position, self.opening_delimiter_position_after, "No characters found after opening delimiter %s." % repr(self.__class__.MY_OPENING_DELIMITER)) return stream.push() seen_escape = False opening_string_token = Tokens.BEGIN_MACRO(self.__class__.MY_OPENING_DELIMITER, self.opening_delimiter_position, self.opening_delimiter_position_after) yield opening_string_token value = "" value_first_position = stream.copy_absolute_position() while True: if stream.next_is_EOF(): stream.pop() if self.__class__.ALLOW_RUNOFF_CLOSING_DELIMITER: position_before_skipping_white_lines = stream.copy_absolute_position() skip_white_lines(stream, readtable) position_before_attempting_to_read_k_chars = stream.copy_absolute_position() k_chars = stream.readn(self.__class__.MY_CLOSING_DELIMITER_LENGTH) if k_chars != self.__class__.MY_CLOSING_DELIMITER: yield Tokens.ERROR(k_chars, position_before_attempting_to_read_k_chars, stream.copy_absolute_position(), "Expected closing string-delimiter «%s», matching opening delimiter «%s» at position %s.%s" % (self.__class__.MY_CLOSING_DELIMITER, self.__class__.MY_OPENING_DELIMITER, self.opening_delimiter_position.nameless_str, "" if k_chars is None else " Found " + repr(k_chars))) return else: value += '\n' yield Tokens.STRING(value, value_first_position, position_before_skipping_white_lines) yield Tokens.END_MACRO(opening_string_token, self.__class__.MY_CLOSING_DELIMITER, stream.absolute_position_of_unread(), stream.copy_absolute_position()) return else: yield Tokens.ERROR("", stream.copy_absolute_position(), stream.copy_absolute_position(), "Expected closing string-delimiter «%s», matching opening delimiter «%s» at position %s." % (self.__class__.MY_CLOSING_DELIMITER, self.__class__.MY_OPENING_DELIMITER, self.opening_delimiter_position.nameless_str)) return char = stream.read() if char == '\\': if seen_escape: value += '\\' seen_escape = False else: seen_escape = True else: if seen_escape: if char in self.__class__.MY_ESCAPE_CHARS: value += self.__class__.MY_ESCAPE_CHARS[char] elif char == self.__class__.MY_INTERPOL_CHAR: value += char elif char == self.__class__.MY_CLOSING_DELIMITER: value += char else: yield Tokens.STRING(value, value_first_position, stream.absolute_position_of_unread()) value = "" value_first_position = stream.copy_absolute_position() yield Tokens.ERROR(char, stream.absolute_position_of_unread(), stream.copy_absolute_position(), "Unknown escape code sequence “%s”." % char) seen_escape = False else: if char == self.MY_INTERPOL_CHAR: yield Tokens.STRING(value, value_first_position, stream.absolute_position_of_unread()) value = "" value_first_position = stream.copy_absolute_position() for token in util.interpolation(self.context): yield token else: value += char last_k_chars = value[-self.__class__.MY_CLOSING_DELIMITER_LENGTH:] if last_k_chars == self.__class__.MY_CLOSING_DELIMITER: value = value[:-self.__class__.MY_CLOSING_DELIMITER_LENGTH] closing_delimiter_first_position = stream.absolute_position_of_unread(self.__class__.MY_CLOSING_DELIMITER_LENGTH) yield Tokens.STRING(value, value_first_position, closing_delimiter_first_position) yield Tokens.END_MACRO(opening_string_token, self.__class__.MY_CLOSING_DELIMITER, closing_delimiter_first_position, stream.copy_absolute_position()) stream.pop() return stream.pop()
def run(self): stream = self.context.stream readtable = self.context.readtable if stream.next_is_EOF(): yield Tokens.ERROR( self.__class__.MY_OPENING_DELIMITER, self.opening_delimiter_position, self.opening_delimiter_position_after, "No characters found after opening delimiter %s." % repr(self.__class__.MY_OPENING_DELIMITER)) return stream.push() seen_escape = False opening_string_token = Tokens.BEGIN_MACRO( self.__class__.MY_OPENING_DELIMITER, self.opening_delimiter_position, self.opening_delimiter_position_after) yield opening_string_token value = "" value_first_position = stream.copy_absolute_position() while True: if stream.next_is_EOF(): stream.pop() if self.__class__.ALLOW_RUNOFF_CLOSING_DELIMITER: position_before_skipping_white_lines = stream.copy_absolute_position( ) skip_white_lines(stream, readtable) position_before_attempting_to_read_k_chars = stream.copy_absolute_position( ) k_chars = stream.readn( self.__class__.MY_CLOSING_DELIMITER_LENGTH) if k_chars != self.__class__.MY_CLOSING_DELIMITER: yield Tokens.ERROR( k_chars, position_before_attempting_to_read_k_chars, stream.copy_absolute_position(), "Expected closing string-delimiter «%s», matching opening delimiter «%s» at position %s.%s" % (self.__class__.MY_CLOSING_DELIMITER, self.__class__.MY_OPENING_DELIMITER, self.opening_delimiter_position.nameless_str, "" if k_chars is None else " Found " + repr(k_chars))) return else: value += '\n' yield Tokens.STRING( value, value_first_position, position_before_skipping_white_lines) yield Tokens.END_MACRO( opening_string_token, self.__class__.MY_CLOSING_DELIMITER, stream.absolute_position_of_unread(), stream.copy_absolute_position()) return else: yield Tokens.ERROR( "", stream.copy_absolute_position(), stream.copy_absolute_position(), "Expected closing string-delimiter «%s», matching opening delimiter «%s» at position %s." % (self.__class__.MY_CLOSING_DELIMITER, self.__class__.MY_OPENING_DELIMITER, self.opening_delimiter_position.nameless_str)) return char = stream.read() if char == '\\': if seen_escape: value += '\\' seen_escape = False else: seen_escape = True else: if seen_escape: if char in self.__class__.MY_ESCAPE_CHARS: value += self.__class__.MY_ESCAPE_CHARS[char] elif char == self.__class__.MY_INTERPOL_CHAR: value += char elif char == self.__class__.MY_CLOSING_DELIMITER: value += char else: yield Tokens.STRING( value, value_first_position, stream.absolute_position_of_unread()) value = "" value_first_position = stream.copy_absolute_position() yield Tokens.ERROR( char, stream.absolute_position_of_unread(), stream.copy_absolute_position(), "Unknown escape code sequence “%s”." % char) seen_escape = False else: if char == self.MY_INTERPOL_CHAR: yield Tokens.STRING( value, value_first_position, stream.absolute_position_of_unread()) value = "" value_first_position = stream.copy_absolute_position() for token in util.interpolation(self.context): yield token else: value += char last_k_chars = value[-self.__class__. MY_CLOSING_DELIMITER_LENGTH:] if last_k_chars == self.__class__.MY_CLOSING_DELIMITER: value = value[:-self.__class__. MY_CLOSING_DELIMITER_LENGTH] closing_delimiter_first_position = stream.absolute_position_of_unread( self.__class__.MY_CLOSING_DELIMITER_LENGTH) yield Tokens.STRING( value, value_first_position, closing_delimiter_first_position) yield Tokens.END_MACRO( opening_string_token, self.__class__.MY_CLOSING_DELIMITER, closing_delimiter_first_position, stream.copy_absolute_position()) stream.pop() return stream.pop()