def invoke(self, tex):
name = []
for t in tex:
if t.nodeType == Command.ELEMENT_NODE and t.nodeName == 'endcsname':
break
name.append(t)
return [EscapeSequence(''.join(name))]
def testExercises(self):
""" Exercises in the TeX book """
# 8.4
tokens = [x for x in TeX().input(r' $x^2$~ \TeX ^^C').itertokens()]
expected = [
MathShift('$'),
Letter('x'),
Superscript('^'),
Other('2'),
MathShift('$'),
EscapeSequence('active::~'),
Space(' '),
EscapeSequence('TeX'),
Other('\x03')
]
self.assertEqual(tokens, expected)
# 8.5
tokens = [x for x in TeX().input('Hi!\n\n\n').itertokens()]
expected = [
Letter('H'),
Letter('i'),
Other('!'),
Space(' '),
EscapeSequence('par')
]
self.assertEqual(tokens, expected)
# 8.6
tokens = [
x for x in TeX().input(r'^^B^^BM^^A^^B^^C^^M^^@\M ').itertokens()
]
expected = [
Other('\x02'),
Other('\x02'),
Letter('M'),
Other('\x01'),
Other('\x02'),
Other('\x03'),
Space(' '),
EscapeSequence('M')
]
self.assertEqual(tokens, expected)
def testParagraph(self):
tokens = [x for x in TeX().input('1\n 2\n \n 3\n').itertokens()]
expected = [
Other('1'),
Space(' '),
Other('2'),
Space(' '),
EscapeSequence('par'),
Other('3'),
Space(' ')
]
self.assertEqual(tokens, expected)
def testTokens(self):
tokens = [x for x in TeX().input(r'{\hskip 36 pt}').itertokens()]
expected = [
BeginGroup('{'),
EscapeSequence('hskip'),
Other('3'),
Other('6'),
Space(' '),
Letter('p'),
Letter('t'),
EndGroup('}')
]
self.assertEqual(tokens, expected)
def invoke(self, tex):
result = Command.invoke(self, tex)
sortkey, key, format = [], [], []
entry = iter(self.attributes['entry'])
current = []
alphanumeric = [Token.CC_OTHER, Token.CC_LETTER, Token.CC_SPACE]
# Parse the index tokens
for tok in entry:
if tok.catcode in alphanumeric:
# Escape character
if tok == '"':
for tok in entry:
current.append(tok)
break
# Entry separator
elif tok == '!':
key.append(current)
if len(sortkey) < len(key):
sortkey.append(current)
current = []
# Sort key separator
elif tok == '@':
sortkey.append(current)
current = []
# Format separator
elif tok == '|':
key.append(current)
if len(sortkey) < len(key):
sortkey.append(current)
current = format
else:
current.append(tok)
continue
# Everything else
current.append(tok)
# Make sure to get the stuff at the end
if not format:
key.append(current)
if len(sortkey) < len(key):
sortkey.append(current)
# Convert the sort keys to strings
for i, item in enumerate(sortkey):
sortkey[i] = tex.expandTokens(item).textContent
# Expand the key tokens
for i, item in enumerate(key):
key[i] = tex.expandTokens(item)
# Get the format element
type = IndexEntry.TYPE_NORMAL
if not format:
format = None
else:
macro = []
while format and format[0].catcode == Token.CC_LETTER:
macro.append(format.pop(0))
if macro:
macro = ''.join(macro)
format.insert(0, EscapeSequence(macro))
if macro == 'see':
type = IndexEntry.TYPE_SEE
elif macro == 'seealso':
type = IndexEntry.TYPE_SEEALSO
format.append(EscapeSequence('index-page-number'))
format = tex.expandTokens(format)
# Store the index information in the document
userdata = self.ownerDocument.userdata
if 'index' not in userdata:
userdata['index'] = []
userdata['index'].append(IndexEntry(key, self, sortkey, format, type))
return result
def testSymbols(self):
tokens = [x for x in TeX().input('\\ { } $ & # ^ _ ~ %').itertokens()]
expected = [
EscapeSequence(' '),
BeginGroup('{'),
Space(' '),
EndGroup('}'),
Space(' '),
MathShift('$'),
Space(' '),
Alignment('&'),
Space(' '),
Parameter('#'),
Space(' '),
Superscript('^'),
Space(' '),
Subscript('_'),
Space(' '),
EscapeSequence('active::~'),
Space(' ')
]
self.assertEqual(tokens, expected)
tokens = [
x for x in TeX().input(
r'\\ \{ \} \$ \& \# \^ \_ \~ \%').itertokens()
]
expected = [
EscapeSequence('\\'),
Space(' '),
EscapeSequence('{'),
Space(' '),
EscapeSequence('}'),
Space(' '),
EscapeSequence('$'),
Space(' '),
EscapeSequence('&'),
Space(' '),
EscapeSequence('#'),
Space(' '),
EscapeSequence('^'),
Space(' '),
EscapeSequence('_'),
Space(' '),
EscapeSequence('~'),
Space(' '),
EscapeSequence('%')
]
self.assertEqual(tokens, expected)
def invoke(self, tex):
"""
We enter verbatim mode by setting all category codes to CC_LETTER
or CC_OTHER. However, we will have to manually scan for the end of the
environment since the tokenizer does not tokenize the end of the
environment as an EscapeSequence Token.
"""
if self.macroMode == Environment.MODE_END:
return
escape = self.ownerDocument.context.categories[0][0]
bgroup = self.ownerDocument.context.categories[1][0]
egroup = self.ownerDocument.context.categories[2][0]
self.ownerDocument.context.push(self)
self.parse(tex)
self.ownerDocument.context.setVerbatimCatcodes()
tokens = [self]
# Get the name of the currently expanding environment
name = self.nodeName
if self.macroMode != Environment.MODE_NONE:
if self.ownerDocument.context.currenvir is not None:
name = self.ownerDocument.context.currenvir
envname = None
if self.ownerDocument.context.currenvir is not None:
envname = self.ownerDocument.context.currenvir
# If we were invoked by a command but should be ended by an
# \end{...}, look for and \end{...} and check if it really contains
# \endverbatim
endpattern3 = list(r'%send%s%s%s' %
(escape, bgroup, envname, egroup))
endlength3 = len(endpattern3)
# If we were invoked by a \begin{...} look for an \end{...}
endpattern = list(r'%send%s%s%s' % (escape, bgroup, name, egroup))
# If we were invoked as a command (i.e. \verbatim) look
# for an end without groupings (i.e. \endverbatim)
endpattern2 = list(r'%send%s' % (escape, name))
endlength = len(endpattern)
endlength2 = len(endpattern2)
# Iterate through tokens until the endpattern is found
for tok in tex:
tokens.append(tok)
if len(tokens) >= endlength:
if tokens[-endlength:] == endpattern:
tokens = tokens[:-endlength]
self.ownerDocument.context.pop(self)
# Expand the end of the macro
end = self.ownerDocument.createElement(name)
end.parentNode = self.parentNode
end.macroMode = Environment.MODE_END
res = end.invoke(tex)
if res is None:
res = [end]
tex.pushTokens(res)
break
if len(tokens) >= endlength2:
if tokens[-endlength2:] == endpattern2:
tokens = tokens[:-endlength2]
self.ownerDocument.context.pop(self)
# Expand the end of the macro
end = self.ownerDocument.createElement(name)
end.parentNode = self.parentNode
end.macroMode = Environment.MODE_END
res = end.invoke(tex)
if res is None:
res = [end]
tex.pushTokens(res)
break
if envname is not None and len(tokens) >= endlength3:
if tokens[-endlength3:] == endpattern3:
tokens = tokens[:-endlength3]
self.ownerDocument.context.pop(self)
# Expand the end of the macro
endenv = self.ownerDocument.createElement(envname)
endenv.parentNode = self.parentNode
endenv.macroMode = Environment.MODE_END
res = endenv.invoke(tex)
end = EscapeSequence('end%s' % name)
if end in res:
tex.pushTokens(res)
break
return tokens
def invoke(self, tex):
text = tex.readArgument()
tex.pushToken(EscapeSequence("footnote"))
return text