def run(self):
stream = self.context.stream
# If the macro tokenizer was called with an unkown opening delimiter sequence, mark it as an error and exit
if self.opening_delimiter != self.__class__.OPENING_DELIMITER:
yield Tokens.ERROR(self.opening_delimiter, self.opening_delimiter_position,
self.opening_delimiter_position_after,
"Comment tokenizer called with unknown opening sequence “%s”" % self.opening_delimiter)
return
stream.push()
seen_escape = False
opening_comment_token = Tokens.BEGIN_MACRO(self.__class__.OPENING_DELIMITER, self.opening_delimiter_position, self.opening_delimiter_position_after)
yield opening_comment_token
value = ""
value_first_position = stream.copy_absolute_position()
while True:
if stream.next_is_EOF():
yield Tokens.COMMENT(value, value_first_position, stream.copy_absolute_position())
yield Tokens.END_MACRO(opening_comment_token, "", stream.copy_absolute_position(), stream.copy_absolute_position())
stream.pop()
return
char = stream.read()
if char == '\\':
if seen_escape: value += '\\'
else: seen_escape = True
else:
if seen_escape:
if char == self.__class__.CLOSING_DELIMITER: value += self.__class__.CLOSING_DELIMITER
elif char == '$': value += '$'
else:
yield Tokens.COMMENT(value, value_first_position, stream.absolute_position_of_unread())
value = ""
value_first_position = stream.copy_absolute_position()
yield Tokens.ERROR(char, stream.absolute_position_of_unread(), stream.copy_absolute_position(),
"Unknown escape code sequence “%s”." % char)
seen_escape = False
else:
if char == self.__class__.CLOSING_DELIMITER:
yield Tokens.COMMENT(value, value_first_position, stream.absolute_position_of_unread())
yield Tokens.END_MACRO(opening_comment_token, self.__class__.CLOSING_DELIMITER, stream.absolute_position_of_unread(), stream.copy_absolute_position())
stream.pop()
return
elif char == '$':
yield Tokens.COMMENT(value, value_first_position, stream.absolute_position_of_unread())
value = ""
value_first_position = stream.copy_absolute_position()
for token in util.interpolation(self.context):
yield token
else: value += char
def run(self):
stream = self.context.stream
# If the macro tokenizer was called with an unkown opening delimiter sequence, mark it as an error and exit
if self.opening_delimiter != self.__class__.OPENING_DELIMITER:
yield Tokens.ERROR(
self.opening_delimiter, self.opening_delimiter_position,
self.opening_delimiter_position_after,
"Comment tokenizer called with unknown opening sequence “%s”" %
self.opening_delimiter)
return
stream.push()
seen_escape = False
opening_comment_token = Tokens.BEGIN_MACRO(
self.__class__.OPENING_DELIMITER, self.opening_delimiter_position,
self.opening_delimiter_position_after)
yield opening_comment_token
value = ""
value_first_position = stream.copy_absolute_position()
while True:
if stream.next_is_EOF():
yield Tokens.COMMENT(value, value_first_position,
stream.copy_absolute_position())
yield Tokens.END_MACRO(opening_comment_token, "",
stream.copy_absolute_position(),
stream.copy_absolute_position())
stream.pop()
return
char = stream.read()
if char == '\\':
if seen_escape: value += '\\'
else: seen_escape = True
else:
if seen_escape:
if char == self.__class__.CLOSING_DELIMITER:
value += self.__class__.CLOSING_DELIMITER
elif char == '$':
value += '$'
else:
yield Tokens.COMMENT(
value, value_first_position,
stream.absolute_position_of_unread())
value = ""
value_first_position = stream.copy_absolute_position()
yield Tokens.ERROR(
char, stream.absolute_position_of_unread(),
stream.copy_absolute_position(),
"Unknown escape code sequence “%s”." % char)
seen_escape = False
else:
if char == self.__class__.CLOSING_DELIMITER:
yield Tokens.COMMENT(
value, value_first_position,
stream.absolute_position_of_unread())
yield Tokens.END_MACRO(
opening_comment_token,
self.__class__.CLOSING_DELIMITER,
stream.absolute_position_of_unread(),
stream.copy_absolute_position())
stream.pop()
return
elif char == '$':
yield Tokens.COMMENT(
value, value_first_position,
stream.absolute_position_of_unread())
value = ""
value_first_position = stream.copy_absolute_position()
for token in util.interpolation(self.context):
yield token
else:
value += char
def run(self):
stream = self.context.stream
readtable = self.context.readtable
if stream.next_is_EOF():
yield Tokens.ERROR(self.__class__.MY_OPENING_DELIMITER, self.opening_delimiter_position,
self.opening_delimiter_position_after,
"No characters found after opening delimiter %s." % repr(self.__class__.MY_OPENING_DELIMITER))
return
stream.push()
seen_escape = False
opening_string_token = Tokens.BEGIN_MACRO(self.__class__.MY_OPENING_DELIMITER,
self.opening_delimiter_position,
self.opening_delimiter_position_after)
yield opening_string_token
value = ""
value_first_position = stream.copy_absolute_position()
while True:
if stream.next_is_EOF():
stream.pop()
if self.__class__.ALLOW_RUNOFF_CLOSING_DELIMITER:
position_before_skipping_white_lines = stream.copy_absolute_position()
skip_white_lines(stream, readtable)
position_before_attempting_to_read_k_chars = stream.copy_absolute_position()
k_chars = stream.readn(self.__class__.MY_CLOSING_DELIMITER_LENGTH)
if k_chars != self.__class__.MY_CLOSING_DELIMITER:
yield Tokens.ERROR(k_chars, position_before_attempting_to_read_k_chars, stream.copy_absolute_position(),
"Expected closing string-delimiter «%s», matching opening delimiter «%s» at position %s.%s" %
(self.__class__.MY_CLOSING_DELIMITER,
self.__class__.MY_OPENING_DELIMITER,
self.opening_delimiter_position.nameless_str,
"" if k_chars is None else " Found " + repr(k_chars)))
return
else:
value += '\n'
yield Tokens.STRING(value, value_first_position, position_before_skipping_white_lines)
yield Tokens.END_MACRO(opening_string_token, self.__class__.MY_CLOSING_DELIMITER, stream.absolute_position_of_unread(), stream.copy_absolute_position())
return
else:
yield Tokens.ERROR("", stream.copy_absolute_position(), stream.copy_absolute_position(),
"Expected closing string-delimiter «%s», matching opening delimiter «%s» at position %s." %
(self.__class__.MY_CLOSING_DELIMITER,
self.__class__.MY_OPENING_DELIMITER,
self.opening_delimiter_position.nameless_str))
return
char = stream.read()
if char == '\\':
if seen_escape:
value += '\\'
seen_escape = False
else: seen_escape = True
else:
if seen_escape:
if char in self.__class__.MY_ESCAPE_CHARS:
value += self.__class__.MY_ESCAPE_CHARS[char]
elif char == self.__class__.MY_INTERPOL_CHAR: value += char
elif char == self.__class__.MY_CLOSING_DELIMITER: value += char
else:
yield Tokens.STRING(value, value_first_position, stream.absolute_position_of_unread())
value = ""
value_first_position = stream.copy_absolute_position()
yield Tokens.ERROR(char, stream.absolute_position_of_unread(), stream.copy_absolute_position(), "Unknown escape code sequence “%s”." % char)
seen_escape = False
else:
if char == self.MY_INTERPOL_CHAR:
yield Tokens.STRING(value, value_first_position, stream.absolute_position_of_unread())
value = ""
value_first_position = stream.copy_absolute_position()
for token in util.interpolation(self.context):
yield token
else:
value += char
last_k_chars = value[-self.__class__.MY_CLOSING_DELIMITER_LENGTH:]
if last_k_chars == self.__class__.MY_CLOSING_DELIMITER:
value = value[:-self.__class__.MY_CLOSING_DELIMITER_LENGTH]
closing_delimiter_first_position = stream.absolute_position_of_unread(self.__class__.MY_CLOSING_DELIMITER_LENGTH)
yield Tokens.STRING(value, value_first_position, closing_delimiter_first_position)
yield Tokens.END_MACRO(opening_string_token, self.__class__.MY_CLOSING_DELIMITER,
closing_delimiter_first_position, stream.copy_absolute_position())
stream.pop()
return
stream.pop()
def run(self):
stream = self.context.stream
readtable = self.context.readtable
if stream.next_is_EOF():
yield Tokens.ERROR(
self.__class__.MY_OPENING_DELIMITER,
self.opening_delimiter_position,
self.opening_delimiter_position_after,
"No characters found after opening delimiter %s." %
repr(self.__class__.MY_OPENING_DELIMITER))
return
stream.push()
seen_escape = False
opening_string_token = Tokens.BEGIN_MACRO(
self.__class__.MY_OPENING_DELIMITER,
self.opening_delimiter_position,
self.opening_delimiter_position_after)
yield opening_string_token
value = ""
value_first_position = stream.copy_absolute_position()
while True:
if stream.next_is_EOF():
stream.pop()
if self.__class__.ALLOW_RUNOFF_CLOSING_DELIMITER:
position_before_skipping_white_lines = stream.copy_absolute_position(
)
skip_white_lines(stream, readtable)
position_before_attempting_to_read_k_chars = stream.copy_absolute_position(
)
k_chars = stream.readn(
self.__class__.MY_CLOSING_DELIMITER_LENGTH)
if k_chars != self.__class__.MY_CLOSING_DELIMITER:
yield Tokens.ERROR(
k_chars,
position_before_attempting_to_read_k_chars,
stream.copy_absolute_position(),
"Expected closing string-delimiter «%s», matching opening delimiter «%s» at position %s.%s"
% (self.__class__.MY_CLOSING_DELIMITER,
self.__class__.MY_OPENING_DELIMITER,
self.opening_delimiter_position.nameless_str,
"" if k_chars is None else " Found " +
repr(k_chars)))
return
else:
value += '\n'
yield Tokens.STRING(
value, value_first_position,
position_before_skipping_white_lines)
yield Tokens.END_MACRO(
opening_string_token,
self.__class__.MY_CLOSING_DELIMITER,
stream.absolute_position_of_unread(),
stream.copy_absolute_position())
return
else:
yield Tokens.ERROR(
"", stream.copy_absolute_position(),
stream.copy_absolute_position(),
"Expected closing string-delimiter «%s», matching opening delimiter «%s» at position %s."
% (self.__class__.MY_CLOSING_DELIMITER,
self.__class__.MY_OPENING_DELIMITER,
self.opening_delimiter_position.nameless_str))
return
char = stream.read()
if char == '\\':
if seen_escape:
value += '\\'
seen_escape = False
else:
seen_escape = True
else:
if seen_escape:
if char in self.__class__.MY_ESCAPE_CHARS:
value += self.__class__.MY_ESCAPE_CHARS[char]
elif char == self.__class__.MY_INTERPOL_CHAR:
value += char
elif char == self.__class__.MY_CLOSING_DELIMITER:
value += char
else:
yield Tokens.STRING(
value, value_first_position,
stream.absolute_position_of_unread())
value = ""
value_first_position = stream.copy_absolute_position()
yield Tokens.ERROR(
char, stream.absolute_position_of_unread(),
stream.copy_absolute_position(),
"Unknown escape code sequence “%s”." % char)
seen_escape = False
else:
if char == self.MY_INTERPOL_CHAR:
yield Tokens.STRING(
value, value_first_position,
stream.absolute_position_of_unread())
value = ""
value_first_position = stream.copy_absolute_position()
for token in util.interpolation(self.context):
yield token
else:
value += char
last_k_chars = value[-self.__class__.
MY_CLOSING_DELIMITER_LENGTH:]
if last_k_chars == self.__class__.MY_CLOSING_DELIMITER:
value = value[:-self.__class__.
MY_CLOSING_DELIMITER_LENGTH]
closing_delimiter_first_position = stream.absolute_position_of_unread(
self.__class__.MY_CLOSING_DELIMITER_LENGTH)
yield Tokens.STRING(
value, value_first_position,
closing_delimiter_first_position)
yield Tokens.END_MACRO(
opening_string_token,
self.__class__.MY_CLOSING_DELIMITER,
closing_delimiter_first_position,
stream.copy_absolute_position())
stream.pop()
return
stream.pop()
