def test_03(self):
"""TestItuToHtmlPhase3.test_03(): Macro with C and a C++ comment."""
myStr = u'#define OBJ_LIKE /* white space */ (1-1) // other \n'
myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr))
myIth.translatePhases123()
myMps = myIth.multiPassString
# Now test
o = [c for c in myMps.genChars()]
self.assertEqual(
[
'#', 'd', 'e', 'f', 'i', 'n', 'e', ' ', 'O', 'B', 'J', '_',
'L', 'I', 'K', 'E', ' ', ' ', ' ', '(', '1', '-', '1', ')',
' ', ' ', '\n'
],
o,
)
n = [c for c in myMps.genWords()]
self.assertEqual(n, [('#', 'preprocessing-op-or-punc'),
('define', 'identifier'), (' ', 'whitespace'),
('OBJ_LIKE', 'identifier'), (' ', 'whitespace'),
('/* white space */', 'C comment'),
(' ', 'whitespace'),
('(', 'preprocessing-op-or-punc'),
('1', 'pp-number'),
('-', 'preprocessing-op-or-punc'),
('1', 'pp-number'),
(')', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('// other ', 'C++ comment'),
('\n', 'whitespace')])
self.assertEqual(51, myMps.idxChar)
def test_01(self):
"""TestItuToHtmlTokenGenLinux.test_01(): linux-3.13/include/linux/init.h."""
myStr = u"""/* Data marked not to be saved by software suspend */
#define __nosavedata __section(.data..nosave)
"""
myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr))
myTokS = [aTok for aTok in myIth.genTokensKeywordPpDirective()]
expTokS = [
('/* Data marked not to be saved by software suspend */',
'C comment'),
('\n', 'whitespace'),
('#', 'preprocessing-op-or-punc'),
('define', 'preprocessing-directive'),
(' ', 'whitespace'),
('__nosavedata', 'identifier'),
(' ', 'whitespace'),
('__section', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('.', 'preprocessing-op-or-punc'),
('data', 'identifier'),
('..', 'preprocessing-op-or-punc'),
('nosave', 'identifier'),
(')', 'preprocessing-op-or-punc'),
('\n', 'whitespace'),
]
self.assertEqual(expTokS, myTokS)
def test_delete_keyword(self):
"""TestItuToHtmlTokenGen: delete as a keyword."""
myStr = u"""delete
"""
myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr))
myTokS = [aTok for aTok in myIth.genTokensKeywordPpDirective()]
expTokS = [('delete', 'keyword'), ('\n', 'whitespace')]
self.assertEqual(expTokS, myTokS)
def test_01(self):
"""TestItuToHtmlLowLevel.test_01(): Empty string, no processing."""
myIth = ItuToTokens.ItuToTokens(io.StringIO(u''))
myMps = myIth.multiPassString
o = [c for c in myMps.genChars()]
self.assertEqual([], o)
n = [c for c in myMps.genWords()]
self.assertEqual([], n)
self.assertEqual([], myMps.currentString)
self.assertEqual(0, myMps.idxChar)
self.assertEqual({}, myMps.idxTypeMap)
def test_new_identifier(self):
"""TestItuToHtmlPhase3.test_20(): new identifier, see TestItuToHtmlTokenGen for the keyword version."""
myStr = u"""new
"""
myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr))
myIth.translatePhases123()
myMps = myIth.multiPassString
wordS = [w for w in myMps.genWords()]
# print()
# print(wordS)
self.assertEqual(wordS, [('new', 'identifier'), ('\n', 'whitespace')])
def test_02(self):
"""TestItuToHtmlLowLevel.test_02(): _translatePhase_1() with empty string."""
myIth = ItuToTokens.ItuToTokens(io.StringIO(u''))
myIth._translatePhase_1()
myMps = myIth.multiPassString
o = [c for c in myMps.genChars()]
self.assertEqual([], o)
n = [c for c in myMps.genWords()]
self.assertEqual([], n)
self.assertEqual([], myMps.currentString)
self.assertEqual(0, myMps.idxChar)
self.assertEqual({}, myMps.idxTypeMap)
def test_02_01(self):
"""TestItuToHtmlTokenGenSpecial.test_02_01(): ISO/IEC 14882:1998(E) 2.12 Operators and punctuators [lex.operators] 'new' is a preprocessing-op-or-punc and keyword."""
myStr = u"""new;
"""
myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr))
myTokS = [aTok for aTok in myIth.genTokensKeywordPpDirective()]
expTokS = [
('new', 'keyword'),
(';', 'preprocessing-op-or-punc'),
('\n', 'whitespace'),
]
self.assertEqual(expTokS, myTokS)
def test_03(self):
"""TestItuToHtmlLowLevel.test_03(): _translatePhase_1() with trigraphs."""
myTestData = [
# Tuples of (input_string, [exp_tokens,], [exp_types,], [current_string])
(
u'??=',
['#'],
[
('??=', 'trigraph'),
],
['#', '', ''],
),
(
u'??=??(',
['#', '['],
[
('??=', 'trigraph'),
('??(', 'trigraph'),
],
['#', '', '', '[', '', ''],
),
(
u'??=??(??)',
['#', '[', ']'],
[
('??=', 'trigraph'),
('??(', 'trigraph'),
('??)', 'trigraph'),
],
['#', '', '', '[', '', '', ']', '', ''],
),
(u'??=??(??)??(??)', list('#[][]'), [
('??=', 'trigraph'),
('??(', 'trigraph'),
('??)', 'trigraph'),
('??(', 'trigraph'),
('??)', 'trigraph'),
],
['#', '', '', '[', '', '', ']', '', '', '[', '', '', ']', '',
'']),
]
for s, eto, ety, cs in myTestData:
myIth = ItuToTokens.ItuToTokens(io.StringIO(s))
myIth._translatePhase_1()
myMps = myIth.multiPassString
o = [c for c in myMps.genChars()]
self.assertEqual(eto, o)
n = [c for c in myMps.genWords()]
self.assertEqual(len(s), myMps.idxChar)
self.assertEqual(ety, n)
self.assertEqual(cs, myMps.currentString)
def test_20(self):
"""TestItuToHtmlPhase3.test_20(): Line continuation."""
myStr = u"""First line \
Second line
"""
myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr))
myIth.translatePhases123()
myMps = myIth.multiPassString
wordS = [w for w in myMps.genWords()]
# print()
# print(wordS)
self.assertEqual(wordS, [('First', 'identifier'), (' ', 'whitespace'),
('line', 'identifier'), (' ', 'whitespace'),
('Second', 'identifier'), (' ', 'whitespace'),
('line', 'identifier'), ('\n', 'whitespace')])
def _initReader(self):
"""Create and return a reader, initialise internals."""
if self._keepGoing:
myDiagnostic = CppDiagnostic.PreprocessDiagnosticKeepGoing()
else:
myDiagnostic = None
try:
myItt = ItuToTokens.ItuToTokens(
theFileObj=self._ituFileObj,
theFileId=self._fpIn,
theDiagnostic=myDiagnostic,
)
except IOError as err:
raise ExceptionItuToHTML(str(err))
self._lineNum = 0
return myItt
def test_02(self):
"""TestItuToHtmlTokenGen.test_02(): Literals."""
myStr = u"""char c = 'c';
long l = 42L;
int i = 42;
float f = 1.234E-27 ;
int o = 0123;
int h = 0xABC;
const char* s = "Hello world";
"""
myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr))
myTokS = [aTok for aTok in myIth.genTokensKeywordPpDirective()]
expTokS = [
('char', 'keyword'), (' ', 'whitespace'), ('c', 'identifier'),
(' ', 'whitespace'), ('=', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ("'c'", 'character-literal'),
(';', 'preprocessing-op-or-punc'), ('\n', 'whitespace'),
('long', 'keyword'), (' ', 'whitespace'), ('l', 'identifier'),
(' ', 'whitespace'), ('=', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('42L', 'pp-number'),
(';', 'preprocessing-op-or-punc'), ('\n', 'whitespace'),
('int', 'keyword'), (' ', 'whitespace'), ('i', 'identifier'),
(' ', 'whitespace'), ('=', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('42', 'pp-number'),
(';', 'preprocessing-op-or-punc'), ('\n', 'whitespace'),
('float', 'keyword'), (' ', 'whitespace'), ('f', 'identifier'),
(' ', 'whitespace'), ('=', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('1.234E-27', 'pp-number'),
(' ', 'whitespace'), (';', 'preprocessing-op-or-punc'),
('\n', 'whitespace'), ('int', 'keyword'), (' ', 'whitespace'),
('o', 'identifier'), (' ', 'whitespace'),
('=', 'preprocessing-op-or-punc'), (' ', 'whitespace'),
('0123', 'pp-number'), (';', 'preprocessing-op-or-punc'),
('\n', 'whitespace'), ('int', 'keyword'), (' ', 'whitespace'),
('h', 'identifier'), (' ', 'whitespace'),
('=', 'preprocessing-op-or-punc'), (' ', 'whitespace'),
('0xABC', 'pp-number'), (';', 'preprocessing-op-or-punc'),
('\n', 'whitespace'), ('const', 'keyword'), (' ', 'whitespace'),
('char', 'keyword'), ('*', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('s', 'identifier'), (' ', 'whitespace'),
('=', 'preprocessing-op-or-punc'), (' ', 'whitespace'),
('"Hello world"', 'string-literal'),
(';', 'preprocessing-op-or-punc'), ('\n', 'whitespace')
]
self.assertEqual(expTokS, myTokS)
def test_01(self):
"""TestItuToHtmlTokenGen.test_01(): Hello world."""
myStr = u"""#include <iostream>
using namespace std;
void main()
{
cout << "Hello World!" << endl;
cout << "Welcome to C++ Programming" << endl;
}
"""
myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr))
myTokS = [aTok for aTok in myIth.genTokensKeywordPpDirective()]
expTokS = [('#', 'preprocessing-op-or-punc'),
('include', 'preprocessing-directive'), (' ', 'whitespace'),
('<', 'preprocessing-op-or-punc'),
('iostream', 'identifier'),
('>', 'preprocessing-op-or-punc'), ('\n\n', 'whitespace'),
('using', 'keyword'), (' ', 'whitespace'),
('namespace', 'keyword'), (' ', 'whitespace'),
('std', 'identifier'), (';', 'preprocessing-op-or-punc'),
('\n\n', 'whitespace'), ('void', 'keyword'),
(' ', 'whitespace'), ('main', 'identifier'),
('(', 'preprocessing-op-or-punc'),
(')', 'preprocessing-op-or-punc'), ('\n', 'whitespace'),
('{', 'preprocessing-op-or-punc'), ('\n ', 'whitespace'),
('cout', 'identifier'), (' ', 'whitespace'),
('<<', 'preprocessing-op-or-punc'), (' ', 'whitespace'),
('"Hello World!"', 'string-literal'), (' ', 'whitespace'),
('<<', 'preprocessing-op-or-punc'), (' ', 'whitespace'),
('endl', 'identifier'), (';', 'preprocessing-op-or-punc'),
('\n ', 'whitespace'), ('cout', 'identifier'),
(' ', 'whitespace'), ('<<', 'preprocessing-op-or-punc'),
(' ', 'whitespace'),
('"Welcome to C++ Programming"', 'string-literal'),
(' ', 'whitespace'), ('<<', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('endl', 'identifier'),
(';', 'preprocessing-op-or-punc'), ('\n', 'whitespace'),
('}', 'preprocessing-op-or-punc'), ('\n', 'whitespace')]
self.assertEqual(expTokS, myTokS)
def test_01(self):
"""TestItuToHtmlTokenGenSpecial.test_01(): Use of $ in a file."""
myStr = u"""# define _ASM_j(cond,dest) _asm jn##cond short $+11 _asm jmp dest
"""
myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr))
myTokS = [aTok for aTok in myIth.genTokensKeywordPpDirective()]
expTokS = [('#', 'preprocessing-op-or-punc'), (' ', 'whitespace'),
('define', 'preprocessing-directive'), (' ', 'whitespace'),
('_ASM_j', 'identifier'), ('(', 'preprocessing-op-or-punc'),
('cond', 'identifier'), (',', 'preprocessing-op-or-punc'),
('dest', 'identifier'), (')', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('_asm', 'identifier'),
(' ', 'whitespace'), ('jn', 'identifier'),
('##', 'preprocessing-op-or-punc'), ('cond', 'identifier'),
(' ', 'whitespace'), ('short', 'keyword'),
(' ', 'whitespace'), ('$', 'identifier'),
('+', 'preprocessing-op-or-punc'), ('11', 'pp-number'),
(' ', 'whitespace'), ('_asm', 'identifier'),
(' ', 'whitespace'), ('jmp', 'identifier'),
(' ', 'whitespace'), ('dest', 'identifier'),
('\n', 'whitespace')]
self.assertEqual(expTokS, myTokS)
def test_02_10(self):
"""TestItuToHtmlTokenGenSpecial.test_02_10(): Use of new after parenthesis."""
myStr = u"""return(new(ELeave) CBufFlat(anExpandSize));
"""
myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr))
myTokS = [aTok for aTok in myIth.genTokensKeywordPpDirective()]
expTokS = [
('return', 'keyword'),
('(', 'preprocessing-op-or-punc'),
('new', 'keyword'),
('(', 'preprocessing-op-or-punc'),
('ELeave', 'identifier'),
(')', 'preprocessing-op-or-punc'),
(' ', 'whitespace'),
('CBufFlat', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('anExpandSize', 'identifier'),
(')', 'preprocessing-op-or-punc'),
(')', 'preprocessing-op-or-punc'),
(';', 'preprocessing-op-or-punc'),
('\n', 'whitespace'),
]
self.assertEqual(expTokS, myTokS)
def test_13(self):
"""TestItuToHtmlPhase3.test_03(): ISO/IEC 9899:1999 (E) 6.10.3.5-5 EXAMPLE 3"""
myStr = u"""#define x 3
#define f(a) f(x * (a))
#undef x
#define x 2
#define g f
#define z z[0]
#define h g(~
#define m(a) a(w)
#define w 0,1
#define t(a) a
#define p() int
#define q(x) x
#define r(x,y) x ## y
#define str(x) # x
f(y+1) + f(f(z)) % t(t(g)(0) + t)(1);
g(x+(3,4)-w) | h 5) & m
(f)^m(m);
p() i[q()] = { q(1), r(2,3), r(4,), r(,5), r(,) };
char c[2][6] = { str(hello), str() };
"""
myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr))
myIth.translatePhases123()
myMps = myIth.multiPassString
wordS = [w for w in myMps.genWords()]
self.assertEqual(wordS, [('#', 'preprocessing-op-or-punc'),
('define', 'identifier'), (' ', 'whitespace'),
('x', 'identifier'), (' ', 'whitespace'),
('3', 'pp-number'), ('\n', 'whitespace'),
('#', 'preprocessing-op-or-punc'),
('define', 'identifier'), (' ', 'whitespace'),
('f', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('a', 'identifier'),
(')', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('f', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('x', 'identifier'), (' ', 'whitespace'),
('*', 'preprocessing-op-or-punc'),
(' ', 'whitespace'),
('(', 'preprocessing-op-or-punc'),
('a', 'identifier'),
(')', 'preprocessing-op-or-punc'),
(')', 'preprocessing-op-or-punc'),
('\n', 'whitespace'),
('#', 'preprocessing-op-or-punc'),
('undef', 'identifier'), (' ', 'whitespace'),
('x', 'identifier'), ('\n', 'whitespace'),
('#', 'preprocessing-op-or-punc'),
('define', 'identifier'), (' ', 'whitespace'),
('x', 'identifier'), (' ', 'whitespace'),
('2', 'pp-number'), ('\n', 'whitespace'),
('#', 'preprocessing-op-or-punc'),
('define', 'identifier'), (' ', 'whitespace'),
('g', 'identifier'), (' ', 'whitespace'),
('f', 'identifier'), ('\n', 'whitespace'),
('#', 'preprocessing-op-or-punc'),
('define', 'identifier'), (' ', 'whitespace'),
('z', 'identifier'), (' ', 'whitespace'),
('z', 'identifier'),
('[', 'preprocessing-op-or-punc'),
('0', 'pp-number'),
(']', 'preprocessing-op-or-punc'),
('\n', 'whitespace'),
('#', 'preprocessing-op-or-punc'),
('define', 'identifier'), (' ', 'whitespace'),
('h', 'identifier'), (' ', 'whitespace'),
('g', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('~', 'preprocessing-op-or-punc'),
('\n', 'whitespace'),
('#', 'preprocessing-op-or-punc'),
('define', 'identifier'), (' ', 'whitespace'),
('m', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('a', 'identifier'),
(')', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('a', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('w', 'identifier'),
(')', 'preprocessing-op-or-punc'),
('\n', 'whitespace'),
('#', 'preprocessing-op-or-punc'),
('define', 'identifier'), (' ', 'whitespace'),
('w', 'identifier'), (' ', 'whitespace'),
('0', 'pp-number'),
(',', 'preprocessing-op-or-punc'),
('1', 'pp-number'), ('\n', 'whitespace'),
('#', 'preprocessing-op-or-punc'),
('define', 'identifier'), (' ', 'whitespace'),
('t', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('a', 'identifier'),
(')', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('a', 'identifier'),
('\n', 'whitespace'),
('#', 'preprocessing-op-or-punc'),
('define', 'identifier'), (' ', 'whitespace'),
('p', 'identifier'),
('(', 'preprocessing-op-or-punc'),
(')', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('int', 'identifier'),
('\n', 'whitespace'),
('#', 'preprocessing-op-or-punc'),
('define', 'identifier'), (' ', 'whitespace'),
('q', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('x', 'identifier'),
(')', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('x', 'identifier'),
('\n', 'whitespace'),
('#', 'preprocessing-op-or-punc'),
('define', 'identifier'), (' ', 'whitespace'),
('r', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('x', 'identifier'),
(',', 'preprocessing-op-or-punc'),
('y', 'identifier'),
(')', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('x', 'identifier'),
(' ', 'whitespace'),
('##', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('y', 'identifier'),
('\n', 'whitespace'),
('#', 'preprocessing-op-or-punc'),
('define', 'identifier'), (' ', 'whitespace'),
('str', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('x', 'identifier'),
(')', 'preprocessing-op-or-punc'),
(' ', 'whitespace'),
('#', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('x', 'identifier'),
('\n', 'whitespace'), ('f', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('y', 'identifier'),
('+', 'preprocessing-op-or-punc'),
('1', 'pp-number'),
(')', 'preprocessing-op-or-punc'),
(' ', 'whitespace'),
('+', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('f', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('f', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('z', 'identifier'),
(')', 'preprocessing-op-or-punc'),
(')', 'preprocessing-op-or-punc'),
(' ', 'whitespace'),
('%', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('t', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('t', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('g', 'identifier'),
(')', 'preprocessing-op-or-punc'),
('(', 'preprocessing-op-or-punc'),
('0', 'pp-number'),
(')', 'preprocessing-op-or-punc'),
(' ', 'whitespace'),
('+', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('t', 'identifier'),
(')', 'preprocessing-op-or-punc'),
('(', 'preprocessing-op-or-punc'),
('1', 'pp-number'),
(')', 'preprocessing-op-or-punc'),
(';', 'preprocessing-op-or-punc'),
('\n', 'whitespace'), ('g', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('x', 'identifier'),
('+', 'preprocessing-op-or-punc'),
('(', 'preprocessing-op-or-punc'),
('3', 'pp-number'),
(',', 'preprocessing-op-or-punc'),
('4', 'pp-number'),
(')', 'preprocessing-op-or-punc'),
('-', 'preprocessing-op-or-punc'),
('w', 'identifier'),
(')', 'preprocessing-op-or-punc'),
(' ', 'whitespace'),
('|', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('h', 'identifier'),
(' ', 'whitespace'), ('5', 'pp-number'),
(')', 'preprocessing-op-or-punc'),
(' ', 'whitespace'),
('&', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('m', 'identifier'),
('\n', 'whitespace'),
('(', 'preprocessing-op-or-punc'),
('f', 'identifier'),
(')', 'preprocessing-op-or-punc'),
('^', 'preprocessing-op-or-punc'),
('m', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('m', 'identifier'),
(')', 'preprocessing-op-or-punc'),
(';', 'preprocessing-op-or-punc'),
('\n', 'whitespace'), ('p', 'identifier'),
('(', 'preprocessing-op-or-punc'),
(')', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('i', 'identifier'),
('[', 'preprocessing-op-or-punc'),
('q', 'identifier'),
('(', 'preprocessing-op-or-punc'),
(')', 'preprocessing-op-or-punc'),
(']', 'preprocessing-op-or-punc'),
(' ', 'whitespace'),
('=', 'preprocessing-op-or-punc'),
(' ', 'whitespace'),
('{', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('q', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('1', 'pp-number'),
(')', 'preprocessing-op-or-punc'),
(',', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('r', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('2', 'pp-number'),
(',', 'preprocessing-op-or-punc'),
('3', 'pp-number'),
(')', 'preprocessing-op-or-punc'),
(',', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('r', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('4', 'pp-number'),
(',', 'preprocessing-op-or-punc'),
(')', 'preprocessing-op-or-punc'),
(',', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('r', 'identifier'),
('(', 'preprocessing-op-or-punc'),
(',', 'preprocessing-op-or-punc'),
('5', 'pp-number'),
(')', 'preprocessing-op-or-punc'),
(',', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('r', 'identifier'),
('(', 'preprocessing-op-or-punc'),
(',', 'preprocessing-op-or-punc'),
(')', 'preprocessing-op-or-punc'),
(' ', 'whitespace'),
('}', 'preprocessing-op-or-punc'),
(';', 'preprocessing-op-or-punc'),
('\n', 'whitespace'), ('char', 'identifier'),
(' ', 'whitespace'), ('c', 'identifier'),
('[', 'preprocessing-op-or-punc'),
('2', 'pp-number'),
(']', 'preprocessing-op-or-punc'),
('[', 'preprocessing-op-or-punc'),
('6', 'pp-number'),
(']', 'preprocessing-op-or-punc'),
(' ', 'whitespace'),
('=', 'preprocessing-op-or-punc'),
(' ', 'whitespace'),
('{', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('str', 'identifier'),
('(', 'preprocessing-op-or-punc'),
('hello', 'identifier'),
(')', 'preprocessing-op-or-punc'),
(',', 'preprocessing-op-or-punc'),
(' ', 'whitespace'), ('str', 'identifier'),
('(', 'preprocessing-op-or-punc'),
(')', 'preprocessing-op-or-punc'),
(' ', 'whitespace'),
('}', 'preprocessing-op-or-punc'),
(';', 'preprocessing-op-or-punc'),
('\n', 'whitespace')])
self.assertEqual(378, myMps.idxChar)
