def test_03(self): """TestItuToHtmlPhase3.test_03(): Macro with C and a C++ comment.""" myStr = u'#define OBJ_LIKE /* white space */ (1-1) // other \n' myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr)) myIth.translatePhases123() myMps = myIth.multiPassString # Now test o = [c for c in myMps.genChars()] self.assertEqual( [ '#', 'd', 'e', 'f', 'i', 'n', 'e', ' ', 'O', 'B', 'J', '_', 'L', 'I', 'K', 'E', ' ', ' ', ' ', '(', '1', '-', '1', ')', ' ', ' ', '\n' ], o, ) n = [c for c in myMps.genWords()] self.assertEqual(n, [('#', 'preprocessing-op-or-punc'), ('define', 'identifier'), (' ', 'whitespace'), ('OBJ_LIKE', 'identifier'), (' ', 'whitespace'), ('/* white space */', 'C comment'), (' ', 'whitespace'), ('(', 'preprocessing-op-or-punc'), ('1', 'pp-number'), ('-', 'preprocessing-op-or-punc'), ('1', 'pp-number'), (')', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('// other ', 'C++ comment'), ('\n', 'whitespace')]) self.assertEqual(51, myMps.idxChar)
def test_01(self): """TestItuToHtmlTokenGenLinux.test_01(): linux-3.13/include/linux/init.h.""" myStr = u"""/* Data marked not to be saved by software suspend */ #define __nosavedata __section(.data..nosave) """ myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr)) myTokS = [aTok for aTok in myIth.genTokensKeywordPpDirective()] expTokS = [ ('/* Data marked not to be saved by software suspend */', 'C comment'), ('\n', 'whitespace'), ('#', 'preprocessing-op-or-punc'), ('define', 'preprocessing-directive'), (' ', 'whitespace'), ('__nosavedata', 'identifier'), (' ', 'whitespace'), ('__section', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('.', 'preprocessing-op-or-punc'), ('data', 'identifier'), ('..', 'preprocessing-op-or-punc'), ('nosave', 'identifier'), (')', 'preprocessing-op-or-punc'), ('\n', 'whitespace'), ] self.assertEqual(expTokS, myTokS)
def test_delete_keyword(self): """TestItuToHtmlTokenGen: delete as a keyword.""" myStr = u"""delete """ myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr)) myTokS = [aTok for aTok in myIth.genTokensKeywordPpDirective()] expTokS = [('delete', 'keyword'), ('\n', 'whitespace')] self.assertEqual(expTokS, myTokS)
def test_01(self): """TestItuToHtmlLowLevel.test_01(): Empty string, no processing.""" myIth = ItuToTokens.ItuToTokens(io.StringIO(u'')) myMps = myIth.multiPassString o = [c for c in myMps.genChars()] self.assertEqual([], o) n = [c for c in myMps.genWords()] self.assertEqual([], n) self.assertEqual([], myMps.currentString) self.assertEqual(0, myMps.idxChar) self.assertEqual({}, myMps.idxTypeMap)
def test_new_identifier(self): """TestItuToHtmlPhase3.test_20(): new identifier, see TestItuToHtmlTokenGen for the keyword version.""" myStr = u"""new """ myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr)) myIth.translatePhases123() myMps = myIth.multiPassString wordS = [w for w in myMps.genWords()] # print() # print(wordS) self.assertEqual(wordS, [('new', 'identifier'), ('\n', 'whitespace')])
def test_02(self): """TestItuToHtmlLowLevel.test_02(): _translatePhase_1() with empty string.""" myIth = ItuToTokens.ItuToTokens(io.StringIO(u'')) myIth._translatePhase_1() myMps = myIth.multiPassString o = [c for c in myMps.genChars()] self.assertEqual([], o) n = [c for c in myMps.genWords()] self.assertEqual([], n) self.assertEqual([], myMps.currentString) self.assertEqual(0, myMps.idxChar) self.assertEqual({}, myMps.idxTypeMap)
def test_02_01(self): """TestItuToHtmlTokenGenSpecial.test_02_01(): ISO/IEC 14882:1998(E) 2.12 Operators and punctuators [lex.operators] 'new' is a preprocessing-op-or-punc and keyword.""" myStr = u"""new; """ myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr)) myTokS = [aTok for aTok in myIth.genTokensKeywordPpDirective()] expTokS = [ ('new', 'keyword'), (';', 'preprocessing-op-or-punc'), ('\n', 'whitespace'), ] self.assertEqual(expTokS, myTokS)
def test_03(self): """TestItuToHtmlLowLevel.test_03(): _translatePhase_1() with trigraphs.""" myTestData = [ # Tuples of (input_string, [exp_tokens,], [exp_types,], [current_string]) ( u'??=', ['#'], [ ('??=', 'trigraph'), ], ['#', '', ''], ), ( u'??=??(', ['#', '['], [ ('??=', 'trigraph'), ('??(', 'trigraph'), ], ['#', '', '', '[', '', ''], ), ( u'??=??(??)', ['#', '[', ']'], [ ('??=', 'trigraph'), ('??(', 'trigraph'), ('??)', 'trigraph'), ], ['#', '', '', '[', '', '', ']', '', ''], ), (u'??=??(??)??(??)', list('#[][]'), [ ('??=', 'trigraph'), ('??(', 'trigraph'), ('??)', 'trigraph'), ('??(', 'trigraph'), ('??)', 'trigraph'), ], ['#', '', '', '[', '', '', ']', '', '', '[', '', '', ']', '', '']), ] for s, eto, ety, cs in myTestData: myIth = ItuToTokens.ItuToTokens(io.StringIO(s)) myIth._translatePhase_1() myMps = myIth.multiPassString o = [c for c in myMps.genChars()] self.assertEqual(eto, o) n = [c for c in myMps.genWords()] self.assertEqual(len(s), myMps.idxChar) self.assertEqual(ety, n) self.assertEqual(cs, myMps.currentString)
def test_20(self): """TestItuToHtmlPhase3.test_20(): Line continuation.""" myStr = u"""First line \ Second line """ myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr)) myIth.translatePhases123() myMps = myIth.multiPassString wordS = [w for w in myMps.genWords()] # print() # print(wordS) self.assertEqual(wordS, [('First', 'identifier'), (' ', 'whitespace'), ('line', 'identifier'), (' ', 'whitespace'), ('Second', 'identifier'), (' ', 'whitespace'), ('line', 'identifier'), ('\n', 'whitespace')])
def _initReader(self): """Create and return a reader, initialise internals.""" if self._keepGoing: myDiagnostic = CppDiagnostic.PreprocessDiagnosticKeepGoing() else: myDiagnostic = None try: myItt = ItuToTokens.ItuToTokens( theFileObj=self._ituFileObj, theFileId=self._fpIn, theDiagnostic=myDiagnostic, ) except IOError as err: raise ExceptionItuToHTML(str(err)) self._lineNum = 0 return myItt
def test_02(self): """TestItuToHtmlTokenGen.test_02(): Literals.""" myStr = u"""char c = 'c'; long l = 42L; int i = 42; float f = 1.234E-27 ; int o = 0123; int h = 0xABC; const char* s = "Hello world"; """ myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr)) myTokS = [aTok for aTok in myIth.genTokensKeywordPpDirective()] expTokS = [ ('char', 'keyword'), (' ', 'whitespace'), ('c', 'identifier'), (' ', 'whitespace'), ('=', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ("'c'", 'character-literal'), (';', 'preprocessing-op-or-punc'), ('\n', 'whitespace'), ('long', 'keyword'), (' ', 'whitespace'), ('l', 'identifier'), (' ', 'whitespace'), ('=', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('42L', 'pp-number'), (';', 'preprocessing-op-or-punc'), ('\n', 'whitespace'), ('int', 'keyword'), (' ', 'whitespace'), ('i', 'identifier'), (' ', 'whitespace'), ('=', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('42', 'pp-number'), (';', 'preprocessing-op-or-punc'), ('\n', 'whitespace'), ('float', 'keyword'), (' ', 'whitespace'), ('f', 'identifier'), (' ', 'whitespace'), ('=', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('1.234E-27', 'pp-number'), (' ', 'whitespace'), (';', 'preprocessing-op-or-punc'), ('\n', 'whitespace'), ('int', 'keyword'), (' ', 'whitespace'), ('o', 'identifier'), (' ', 'whitespace'), ('=', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('0123', 'pp-number'), (';', 'preprocessing-op-or-punc'), ('\n', 'whitespace'), ('int', 'keyword'), (' ', 'whitespace'), ('h', 'identifier'), (' ', 'whitespace'), ('=', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('0xABC', 'pp-number'), (';', 'preprocessing-op-or-punc'), ('\n', 'whitespace'), ('const', 'keyword'), (' ', 'whitespace'), ('char', 'keyword'), ('*', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('s', 'identifier'), (' ', 'whitespace'), ('=', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('"Hello world"', 'string-literal'), (';', 'preprocessing-op-or-punc'), ('\n', 'whitespace') ] self.assertEqual(expTokS, myTokS)
def test_01(self): """TestItuToHtmlTokenGen.test_01(): Hello world.""" myStr = u"""#include <iostream> using namespace std; void main() { cout << "Hello World!" << endl; cout << "Welcome to C++ Programming" << endl; } """ myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr)) myTokS = [aTok for aTok in myIth.genTokensKeywordPpDirective()] expTokS = [('#', 'preprocessing-op-or-punc'), ('include', 'preprocessing-directive'), (' ', 'whitespace'), ('<', 'preprocessing-op-or-punc'), ('iostream', 'identifier'), ('>', 'preprocessing-op-or-punc'), ('\n\n', 'whitespace'), ('using', 'keyword'), (' ', 'whitespace'), ('namespace', 'keyword'), (' ', 'whitespace'), ('std', 'identifier'), (';', 'preprocessing-op-or-punc'), ('\n\n', 'whitespace'), ('void', 'keyword'), (' ', 'whitespace'), ('main', 'identifier'), ('(', 'preprocessing-op-or-punc'), (')', 'preprocessing-op-or-punc'), ('\n', 'whitespace'), ('{', 'preprocessing-op-or-punc'), ('\n ', 'whitespace'), ('cout', 'identifier'), (' ', 'whitespace'), ('<<', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('"Hello World!"', 'string-literal'), (' ', 'whitespace'), ('<<', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('endl', 'identifier'), (';', 'preprocessing-op-or-punc'), ('\n ', 'whitespace'), ('cout', 'identifier'), (' ', 'whitespace'), ('<<', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('"Welcome to C++ Programming"', 'string-literal'), (' ', 'whitespace'), ('<<', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('endl', 'identifier'), (';', 'preprocessing-op-or-punc'), ('\n', 'whitespace'), ('}', 'preprocessing-op-or-punc'), ('\n', 'whitespace')] self.assertEqual(expTokS, myTokS)
def test_01(self): """TestItuToHtmlTokenGenSpecial.test_01(): Use of $ in a file.""" myStr = u"""# define _ASM_j(cond,dest) _asm jn##cond short $+11 _asm jmp dest """ myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr)) myTokS = [aTok for aTok in myIth.genTokensKeywordPpDirective()] expTokS = [('#', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('define', 'preprocessing-directive'), (' ', 'whitespace'), ('_ASM_j', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('cond', 'identifier'), (',', 'preprocessing-op-or-punc'), ('dest', 'identifier'), (')', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('_asm', 'identifier'), (' ', 'whitespace'), ('jn', 'identifier'), ('##', 'preprocessing-op-or-punc'), ('cond', 'identifier'), (' ', 'whitespace'), ('short', 'keyword'), (' ', 'whitespace'), ('$', 'identifier'), ('+', 'preprocessing-op-or-punc'), ('11', 'pp-number'), (' ', 'whitespace'), ('_asm', 'identifier'), (' ', 'whitespace'), ('jmp', 'identifier'), (' ', 'whitespace'), ('dest', 'identifier'), ('\n', 'whitespace')] self.assertEqual(expTokS, myTokS)
def test_02_10(self): """TestItuToHtmlTokenGenSpecial.test_02_10(): Use of new after parenthesis.""" myStr = u"""return(new(ELeave) CBufFlat(anExpandSize)); """ myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr)) myTokS = [aTok for aTok in myIth.genTokensKeywordPpDirective()] expTokS = [ ('return', 'keyword'), ('(', 'preprocessing-op-or-punc'), ('new', 'keyword'), ('(', 'preprocessing-op-or-punc'), ('ELeave', 'identifier'), (')', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('CBufFlat', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('anExpandSize', 'identifier'), (')', 'preprocessing-op-or-punc'), (')', 'preprocessing-op-or-punc'), (';', 'preprocessing-op-or-punc'), ('\n', 'whitespace'), ] self.assertEqual(expTokS, myTokS)
def test_13(self): """TestItuToHtmlPhase3.test_03(): ISO/IEC 9899:1999 (E) 6.10.3.5-5 EXAMPLE 3""" myStr = u"""#define x 3 #define f(a) f(x * (a)) #undef x #define x 2 #define g f #define z z[0] #define h g(~ #define m(a) a(w) #define w 0,1 #define t(a) a #define p() int #define q(x) x #define r(x,y) x ## y #define str(x) # x f(y+1) + f(f(z)) % t(t(g)(0) + t)(1); g(x+(3,4)-w) | h 5) & m (f)^m(m); p() i[q()] = { q(1), r(2,3), r(4,), r(,5), r(,) }; char c[2][6] = { str(hello), str() }; """ myIth = ItuToTokens.ItuToTokens(io.StringIO(myStr)) myIth.translatePhases123() myMps = myIth.multiPassString wordS = [w for w in myMps.genWords()] self.assertEqual(wordS, [('#', 'preprocessing-op-or-punc'), ('define', 'identifier'), (' ', 'whitespace'), ('x', 'identifier'), (' ', 'whitespace'), ('3', 'pp-number'), ('\n', 'whitespace'), ('#', 'preprocessing-op-or-punc'), ('define', 'identifier'), (' ', 'whitespace'), ('f', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('a', 'identifier'), (')', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('f', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('x', 'identifier'), (' ', 'whitespace'), ('*', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('(', 'preprocessing-op-or-punc'), ('a', 'identifier'), (')', 'preprocessing-op-or-punc'), (')', 'preprocessing-op-or-punc'), ('\n', 'whitespace'), ('#', 'preprocessing-op-or-punc'), ('undef', 'identifier'), (' ', 'whitespace'), ('x', 'identifier'), ('\n', 'whitespace'), ('#', 'preprocessing-op-or-punc'), ('define', 'identifier'), (' ', 'whitespace'), ('x', 'identifier'), (' ', 'whitespace'), ('2', 'pp-number'), ('\n', 'whitespace'), ('#', 'preprocessing-op-or-punc'), ('define', 'identifier'), (' ', 'whitespace'), ('g', 'identifier'), (' ', 'whitespace'), ('f', 'identifier'), ('\n', 'whitespace'), ('#', 'preprocessing-op-or-punc'), ('define', 'identifier'), (' ', 'whitespace'), ('z', 'identifier'), (' ', 'whitespace'), ('z', 'identifier'), ('[', 'preprocessing-op-or-punc'), ('0', 'pp-number'), (']', 'preprocessing-op-or-punc'), ('\n', 'whitespace'), ('#', 'preprocessing-op-or-punc'), ('define', 'identifier'), (' ', 'whitespace'), ('h', 'identifier'), (' ', 'whitespace'), ('g', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('~', 'preprocessing-op-or-punc'), ('\n', 'whitespace'), ('#', 'preprocessing-op-or-punc'), ('define', 'identifier'), (' ', 'whitespace'), ('m', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('a', 'identifier'), (')', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('a', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('w', 'identifier'), (')', 'preprocessing-op-or-punc'), ('\n', 'whitespace'), ('#', 'preprocessing-op-or-punc'), ('define', 'identifier'), (' ', 'whitespace'), ('w', 'identifier'), (' ', 'whitespace'), ('0', 'pp-number'), (',', 'preprocessing-op-or-punc'), ('1', 'pp-number'), ('\n', 'whitespace'), ('#', 'preprocessing-op-or-punc'), ('define', 'identifier'), (' ', 'whitespace'), ('t', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('a', 'identifier'), (')', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('a', 'identifier'), ('\n', 'whitespace'), ('#', 'preprocessing-op-or-punc'), ('define', 'identifier'), (' ', 'whitespace'), ('p', 'identifier'), ('(', 'preprocessing-op-or-punc'), (')', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('int', 'identifier'), ('\n', 'whitespace'), ('#', 'preprocessing-op-or-punc'), ('define', 'identifier'), (' ', 'whitespace'), ('q', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('x', 'identifier'), (')', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('x', 'identifier'), ('\n', 'whitespace'), ('#', 'preprocessing-op-or-punc'), ('define', 'identifier'), (' ', 'whitespace'), ('r', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('x', 'identifier'), (',', 'preprocessing-op-or-punc'), ('y', 'identifier'), (')', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('x', 'identifier'), (' ', 'whitespace'), ('##', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('y', 'identifier'), ('\n', 'whitespace'), ('#', 'preprocessing-op-or-punc'), ('define', 'identifier'), (' ', 'whitespace'), ('str', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('x', 'identifier'), (')', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('#', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('x', 'identifier'), ('\n', 'whitespace'), ('f', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('y', 'identifier'), ('+', 'preprocessing-op-or-punc'), ('1', 'pp-number'), (')', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('+', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('f', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('f', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('z', 'identifier'), (')', 'preprocessing-op-or-punc'), (')', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('%', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('t', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('t', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('g', 'identifier'), (')', 'preprocessing-op-or-punc'), ('(', 'preprocessing-op-or-punc'), ('0', 'pp-number'), (')', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('+', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('t', 'identifier'), (')', 'preprocessing-op-or-punc'), ('(', 'preprocessing-op-or-punc'), ('1', 'pp-number'), (')', 'preprocessing-op-or-punc'), (';', 'preprocessing-op-or-punc'), ('\n', 'whitespace'), ('g', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('x', 'identifier'), ('+', 'preprocessing-op-or-punc'), ('(', 'preprocessing-op-or-punc'), ('3', 'pp-number'), (',', 'preprocessing-op-or-punc'), ('4', 'pp-number'), (')', 'preprocessing-op-or-punc'), ('-', 'preprocessing-op-or-punc'), ('w', 'identifier'), (')', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('|', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('h', 'identifier'), (' ', 'whitespace'), ('5', 'pp-number'), (')', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('&', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('m', 'identifier'), ('\n', 'whitespace'), ('(', 'preprocessing-op-or-punc'), ('f', 'identifier'), (')', 'preprocessing-op-or-punc'), ('^', 'preprocessing-op-or-punc'), ('m', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('m', 'identifier'), (')', 'preprocessing-op-or-punc'), (';', 'preprocessing-op-or-punc'), ('\n', 'whitespace'), ('p', 'identifier'), ('(', 'preprocessing-op-or-punc'), (')', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('i', 'identifier'), ('[', 'preprocessing-op-or-punc'), ('q', 'identifier'), ('(', 'preprocessing-op-or-punc'), (')', 'preprocessing-op-or-punc'), (']', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('=', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('{', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('q', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('1', 'pp-number'), (')', 'preprocessing-op-or-punc'), (',', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('r', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('2', 'pp-number'), (',', 'preprocessing-op-or-punc'), ('3', 'pp-number'), (')', 'preprocessing-op-or-punc'), (',', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('r', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('4', 'pp-number'), (',', 'preprocessing-op-or-punc'), (')', 'preprocessing-op-or-punc'), (',', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('r', 'identifier'), ('(', 'preprocessing-op-or-punc'), (',', 'preprocessing-op-or-punc'), ('5', 'pp-number'), (')', 'preprocessing-op-or-punc'), (',', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('r', 'identifier'), ('(', 'preprocessing-op-or-punc'), (',', 'preprocessing-op-or-punc'), (')', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('}', 'preprocessing-op-or-punc'), (';', 'preprocessing-op-or-punc'), ('\n', 'whitespace'), ('char', 'identifier'), (' ', 'whitespace'), ('c', 'identifier'), ('[', 'preprocessing-op-or-punc'), ('2', 'pp-number'), (']', 'preprocessing-op-or-punc'), ('[', 'preprocessing-op-or-punc'), ('6', 'pp-number'), (']', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('=', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('{', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('str', 'identifier'), ('(', 'preprocessing-op-or-punc'), ('hello', 'identifier'), (')', 'preprocessing-op-or-punc'), (',', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('str', 'identifier'), ('(', 'preprocessing-op-or-punc'), (')', 'preprocessing-op-or-punc'), (' ', 'whitespace'), ('}', 'preprocessing-op-or-punc'), (';', 'preprocessing-op-or-punc'), ('\n', 'whitespace')]) self.assertEqual(378, myMps.idxChar)