def test_tokensStr_01(self): """global function tokensStr() test multiple tokens.""" myTokS = [ PpToken.PpToken('f', 'identifier'), PpToken.PpToken('g', 'identifier'), ] self.assertEqual('fg', PpToken.tokensStr(myTokS))
def test_01(self): """TestPpTokenCount __iadd__ simple""" #print self._ptc_1.inc(PpToken.PpToken('a', 'identifier'), True) self._ptc_2.inc(PpToken.PpToken('a', 'identifier'), True) #=============================================================================== # print '1 True:' # pprint.pprint([x for x in self._ptc_1.tokenTypesAndCounts(True, allPossibleTypes=False)]) # print '1 False:' # pprint.pprint([x for x in self._ptc_1.tokenTypesAndCounts(False, allPossibleTypes=False)]) # print # print '2 True:' # pprint.pprint([x for x in self._ptc_2.tokenTypesAndCounts(True, allPossibleTypes=False)]) # print '2 False:' # pprint.pprint([x for x in self._ptc_2.tokenTypesAndCounts(False, allPossibleTypes=False)]) #=============================================================================== # self._ptc_1 += self._ptc_2 #=============================================================================== # print # print 'Merged True:' # pprint.pprint([x for x in self._ptc_1.tokenTypesAndCounts(True, allPossibleTypes=False)]) # print 'Merged False:' # pprint.pprint([x for x in self._ptc_1.tokenTypesAndCounts(False, allPossibleTypes=False)]) #=============================================================================== self.assertEqual([ x for x in self._ptc_1.tokenTypesAndCounts(True, allPossibleTypes=False) ], [('identifier', 2)]) self.assertEqual([ x for x in self._ptc_1.tokenTypesAndCounts(False, allPossibleTypes=False) ], [('identifier', 2)])
def testMerge_01(self): """PpToken() test merge() type='concat' with ws/identifier.""" myObj_00 = PpToken.PpToken(' ', 'whitespace') self.assertEqual(' ', myObj_00.t) self.assertEqual('whitespace', myObj_00.tt) self.assertEqual(False, myObj_00.canReplace) self.assertEqual(False, myObj_00.prevWs) self.assertEqual(True, myObj_00.isWs()) myObj_01 = PpToken.PpToken('f', 'identifier') self.assertEqual('f', myObj_01.t) self.assertEqual('identifier', myObj_01.tt) self.assertEqual(True, myObj_01.canReplace) self.assertEqual(False, myObj_01.prevWs) self.assertEqual(False, myObj_01.isWs()) #self.assertRaises( # PpToken.ExceptionCpipTokenIllegalMerge, # myObj_00.merge, # myObj_01) #self.assertRaises( # PpToken.ExceptionCpipTokenIllegalMerge, # myObj_01.merge, # myObj_00) myObj_00.merge(myObj_01) self.assertEqual(' f', myObj_00.t) self.assertEqual('concat', myObj_00.tt) self.assertEqual(False, myObj_00.canReplace) self.assertEqual(False, myObj_00.prevWs) self.assertEqual(False, myObj_00.isWs())
def defined(self, theTtt, flagInvert, theFileLineCol=None): """If the PpToken theTtt is an identifier that is currently defined then this returns 1 as a PpToken, 0 as a PpToken otherwise. If the macro exists in the environment its reference count is incremented. See: ISO/IEC 9899:1999 (E) 6.10.1. theFileLineCol is a FileLocation.FileLineCol object.""" # If myTtt is newline then raise as this is #if defined\n if not theTtt.isIdentifier(): raise ExceptionMacroEnv( 'defined() on non-identifier but: %s' % theTtt) try: self._defineMap[theTtt.t].incRefCount(theFileLineCol) if flagInvert: return PpToken.PpToken('0', 'pp-number') return PpToken.PpToken('1', 'pp-number') except KeyError: try: self._ifDefAbsentMacros[theTtt.t].append(theFileLineCol) except KeyError: self._ifDefAbsentMacros[theTtt.t] = [theFileLineCol,] # No macro of this name defined if flagInvert: return PpToken.PpToken('1', 'pp-number') return PpToken.PpToken('0', 'pp-number')
def test_05(self): """TestPpTokenEvalConstExpr.test_05(): Integer literals, unsigned/long, short strings.""" myTok = PpToken.PpToken('1ul', 'pp-number') self.assertEqual('1', myTok.evalConstExpr()) myTok = PpToken.PpToken('1U', 'pp-number') self.assertEqual('1', myTok.evalConstExpr()) myTok = PpToken.PpToken('', 'pp-number') self.assertEqual('', myTok.evalConstExpr())
def test_tokensStr_02(self): """global function tokensStr() test single token, long form.""" myTokS = [ PpToken.PpToken('f', 'identifier'), ] self.assertEqual( #'"f", identifier, True, False, False', 'PpToken(t="f", tt=identifier, line=True, prev=False, ?=False)', PpToken.tokensStr(myTokS, shortForm=False))
def test_01(self): """TestPpTokenEvalConstExpr.test_01(): Integer literals.""" myTok = PpToken.PpToken('', 'pp-number') self.assertEqual('', myTok.evalConstExpr()) myTok = PpToken.PpToken('1', 'pp-number') self.assertEqual('1', myTok.evalConstExpr()) myTok = PpToken.PpToken('12', 'pp-number') self.assertEqual('12', myTok.evalConstExpr()) myTok = PpToken.PpToken('123', 'pp-number') self.assertEqual('123', myTok.evalConstExpr())
def test_06(self): """TestPpTokenEvalConstExpr.test_06(): Integer literals, Hex and unsigned/long.""" myTok = PpToken.PpToken('0xFF', 'pp-number') self.assertEqual('0xFF', myTok.evalConstExpr()) myTok = PpToken.PpToken('0XAF', 'pp-number') self.assertEqual('0XAF', myTok.evalConstExpr()) myTok = PpToken.PpToken('0xFFul', 'pp-number') self.assertEqual('0xFF', myTok.evalConstExpr()) myTok = PpToken.PpToken('0XAFul', 'pp-number') self.assertEqual('0XAF', myTok.evalConstExpr())
def test_07(self): """TestPpTokenEvalConstExpr.test_07(): Integer literals, Octal and unsigned/long.""" myTok = PpToken.PpToken('012', 'pp-number') self.assertEqual('012', myTok.evalConstExpr()) myTok = PpToken.PpToken('017', 'pp-number') self.assertEqual('017', myTok.evalConstExpr()) myTok = PpToken.PpToken('017uL', 'pp-number') self.assertEqual('017', myTok.evalConstExpr()) myTok = PpToken.PpToken('017l', 'pp-number') self.assertEqual('017', myTok.evalConstExpr())
def test_STDC_FP_CONTRACT_ON_fail_02(self): """TestPragmaHandlerSTDC: Test FP_CONTRACT ON no ON.""" myH = PragmaHandler.PragmaHandlerSTDC() myTokS = [ PpToken.PpToken('STDC', 'identifier'), PpToken.PpToken(' ', 'whitespace'), PpToken.PpToken('FP_CONTRACT', 'identifier'), PpToken.PpToken(' ', 'whitespace'), PpToken.PpToken('\n', 'whitespace'), ] self.assertEqual(myH.pragma(myTokS), '')
def test_00(self): """TestPpTokenEvalConstExpr.test_00(): Floating point literals.""" myTok = PpToken.PpToken('1.23e4', 'pp-number') self.assertEqual('1.23e4', myTok.evalConstExpr()) myTok = PpToken.PpToken('1.23e4f', 'pp-number') self.assertEqual('1.23e4', myTok.evalConstExpr()) myTok = PpToken.PpToken('1.23e4F', 'pp-number') self.assertEqual('1.23e4', myTok.evalConstExpr()) myTok = PpToken.PpToken('1.23e4l', 'pp-number') self.assertEqual('1.23e4', myTok.evalConstExpr()) myTok = PpToken.PpToken('1.23e4L', 'pp-number') self.assertEqual('1.23e4', myTok.evalConstExpr())
def test_10(self): """TestPpTokenEvalConstExpr.test_10(): Identifiers.""" myTok = PpToken.PpToken('ABC', 'identifier') self.assertEqual('0', myTok.evalConstExpr()) myTok = PpToken.PpToken('true', 'identifier') self.assertEqual('True', myTok.evalConstExpr()) myTok = PpToken.PpToken('false', 'identifier') self.assertEqual('False', myTok.evalConstExpr()) myTok = PpToken.PpToken('&&', 'preprocessing-op-or-punc') self.assertEqual('and', myTok.evalConstExpr()) myTok = PpToken.PpToken('||', 'preprocessing-op-or-punc') self.assertEqual('or', myTok.evalConstExpr())
def test_02(self): """TestPpLexerSimpleReplace: Test with "spam\\n" * 1000, #define spam eggs and #define eggs chips.""" myCont = '#define spam eggs\n' myCont += '#define eggs chips\n' myCont += 'spam\n' * self.NUM_TESTS myL = self.simpleLexWithContent(myCont) myToks, myTime = self.runLex(myL) #logging.info('Rate: %6.1f tokens/second.' % (len(myToks) / myTime)) self.assertEqual( [ PpToken.PpToken('\n', 'whitespace'), PpToken.PpToken('\n', 'whitespace'), ] + self.stringToTokens('chips\n' * self.NUM_TESTS), myToks, )
def testInit_01(self): """PpToken() ctor good: PpToken('spam', 1)""" myObj = PpToken.PpToken('spam', 1) self.assertEqual('spam', myObj.t) self.assertEqual('identifier', myObj.tt) self.assertEqual(True, myObj.canReplace) self.assertEqual(False, myObj.prevWs)
def testWs_00(self): """PpToken() test not whitespace with isWs()""" myObj = PpToken.PpToken('spam', 'identifier') self.assertEqual('spam', myObj.t) self.assertEqual('identifier', myObj.tt) self.assertEqual(True, myObj.canReplace) self.assertEqual(False, myObj.prevWs) self.assertEqual(False, myObj.isWs())
def testWs_01(self): """PpToken() test is whitespace with isWs()""" myObj = PpToken.PpToken(' ', 'whitespace') self.assertEqual(' ', myObj.t) self.assertEqual('whitespace', myObj.tt) self.assertEqual(False, myObj.canReplace) self.assertEqual(False, myObj.prevWs) self.assertEqual(True, myObj.isWs())
def testInit_02_01(self): """PpToken() ctor good: PpToken('<spam>', 0)""" myObj = PpToken.PpToken('<spam>', 0) self.assertEqual('<spam>', myObj.t) self.assertEqual('header-name', myObj.tt) self.assertEqual(False, myObj.canReplace) self.assertNotEqual(True, myObj.canReplace) self.assertEqual(False, myObj.prevWs)
def testTokenGenerator(self): """ListAsGenerator: list of PpToken(token, token_type).""" myList = [ PpToken.PpToken('#', 'preprocessing-op-or-punc'), PpToken.PpToken('define', 'identifier'), PpToken.PpToken(' ', 'whitespace'), PpToken.PpToken('BAR', 'identifier'), PpToken.PpToken('\n', 'whitespace'), ] myG = next(ListGen.ListAsGenerator(myList)) self.assertEqual(PpToken.PpToken('#', 'preprocessing-op-or-punc'), next(myG)) self.assertEqual(PpToken.PpToken('define', 'identifier'), next(myG)) self.assertEqual(PpToken.PpToken(' ', 'whitespace'), next(myG)) self.assertEqual(PpToken.PpToken('BAR', 'identifier'), next(myG)) self.assertEqual(PpToken.PpToken('\n', 'whitespace'), next(myG)) self.assertRaises(StopIteration, next, myG)
def testSetReplace_00(self): """PpToken() canReplace success: for ('spam', 'identifier')""" myObj = PpToken.PpToken('spam', 'identifier') self.assertEqual('spam', myObj.t) self.assertEqual('identifier', myObj.tt) self.assertEqual(True, myObj.canReplace) self.assertEqual(False, myObj.prevWs) myObj.canReplace = False self.assertEqual(False, myObj.canReplace)
def defined(self, theTtt, flagInvert, theFileLineCol=None): """If the PpToken theTtt is an identifier that is currently defined then this returns 1 as a PpToken, 0 as a PpToken otherwise. If the macro exists in the environment its reference count is incremented. *theFileLineCol* Is a :py:class:`.FileLocation.FileLineCol object`. See: :title-reference:`ISO/IEC 9899:1999 (E) 6.10.1.` :param theTtt: The token. :type theTtt: :py:class:`cpip.core.PpToken.PpToken` :param flagInvert: Invert the test. :type flagInvert: ``bool`` :param theFileLineCol: File location. :type theFileLineCol: ``cpip.core.FileLocation.FileLineCol([str, int, int])`` :returns: :py:class:`cpip.core.PpToken.PpToken` -- A token that is either 0 or 1. :raises: ``KeyError`` """ # If myTtt is newline then raise as this is #if defined\n if not theTtt.isIdentifier(): raise ExceptionMacroEnv('defined() on non-identifier but: %s' % theTtt) try: self._defineMap[theTtt.t].incRefCount(theFileLineCol) if flagInvert: return PpToken.PpToken('0', 'pp-number') return PpToken.PpToken('1', 'pp-number') except KeyError: try: self._ifDefAbsentMacros[theTtt.t].append(theFileLineCol) except KeyError: self._ifDefAbsentMacros[theTtt.t] = [ theFileLineCol, ] # No macro of this name defined if flagInvert: return PpToken.PpToken('1', 'pp-number') return PpToken.PpToken('0', 'pp-number')
def test_shrinkWs_01(self): """PpToken() test shrinkWs() fails.""" myObj_00 = PpToken.PpToken('f', 'identifier') self.assertEqual('f', myObj_00.t) self.assertEqual('identifier', myObj_00.tt) self.assertEqual(True, myObj_00.canReplace) self.assertEqual(False, myObj_00.prevWs) self.assertEqual(False, myObj_00.isWs()) self.assertRaises(PpToken.ExceptionCpipTokenIllegalOperation, myObj_00.shrinkWs)
def _reexamineLevel(self, theLevel): myCont, myStart, myEnd = self.retObjectMacros(theLevel) myCont += ('%s\n' % myStart) * self.NUM_TESTS myL = self.simpleLexWithContent(myCont) myToks, myTime = self.runLex(myL) myExp = [] for i in range(theLevel): myExp.append(PpToken.PpToken('\n', 'whitespace')) myExp += self.stringToTokens(('%s\n' % myEnd) * self.NUM_TESTS) self.assertEqual(myExp, myToks)
def testMerge_02(self): """PpToken() test merge() succeeds with identifier/identifier.""" myObj_00 = PpToken.PpToken('f', 'identifier') self.assertEqual('f', myObj_00.t) self.assertEqual('identifier', myObj_00.tt) self.assertEqual(True, myObj_00.canReplace) self.assertEqual(False, myObj_00.prevWs) self.assertEqual(False, myObj_00.isWs()) myObj_01 = PpToken.PpToken('g', 'identifier') self.assertEqual('g', myObj_01.t) self.assertEqual('identifier', myObj_01.tt) self.assertEqual(True, myObj_01.canReplace) self.assertEqual(False, myObj_01.prevWs) self.assertEqual(False, myObj_01.isWs()) myObj_00.merge(myObj_01) self.assertEqual('fg', myObj_00.t) self.assertEqual('identifier', myObj_00.tt) self.assertEqual(True, myObj_00.canReplace) self.assertEqual(False, myObj_00.prevWs) self.assertEqual(False, myObj_00.isWs())
def testMerge_00(self): """PpToken() test merge() succeeds with whitespace.""" myObj_00 = PpToken.PpToken(' ', 'whitespace') self.assertEqual(' ', myObj_00.t) self.assertEqual('whitespace', myObj_00.tt) self.assertEqual(False, myObj_00.canReplace) self.assertEqual(False, myObj_00.prevWs) self.assertEqual(True, myObj_00.isWs()) myObj_01 = PpToken.PpToken(' ', 'whitespace') self.assertEqual(' ', myObj_01.t) self.assertEqual('whitespace', myObj_01.tt) self.assertEqual(False, myObj_01.canReplace) self.assertEqual(False, myObj_01.prevWs) self.assertEqual(True, myObj_01.isWs()) myObj_00.merge(myObj_01) self.assertEqual(' ', myObj_00.t) self.assertEqual('whitespace', myObj_00.tt) self.assertEqual(False, myObj_00.canReplace) self.assertEqual(False, myObj_00.prevWs) self.assertEqual(True, myObj_00.isWs())
def test_isCond_02(self): """isCond can be not be set to False.""" myTok = PpToken.PpToken('f', 'identifier') self.assertEqual(False, myTok.isCond) myTok.setIsCond() self.assertEqual(True, myTok.isCond) try: myTok.isCond = False self.fail('No AttributeError raised.') except AttributeError: pass
def testPrevWs_00(self): """PpToken() can set/clear prevWs for ('spam', 'identifier')""" myObj = PpToken.PpToken('spam', 'identifier') self.assertEqual('spam', myObj.t) self.assertEqual('identifier', myObj.tt) self.assertEqual(True, myObj.canReplace) self.assertEqual(False, myObj.prevWs) myObj.prevWs = True self.assertEqual(True, myObj.prevWs) myObj.prevWs = False self.assertEqual(False, myObj.prevWs)
def testInit_03(self): """PpToken() ctor good: with non-identifier types (ints).""" for i in PpToken.LEX_PPTOKEN_TYPE_ENUM_RANGE: if PpToken.ENUM_NAME[i] != 'identifier': myObj = PpToken.PpToken('', i) self.assertEqual('', myObj.t) self.assertEqual(PpToken.ENUM_NAME[i], myObj.tt) self.assertEqual(False, myObj.canReplace) self.assertNotEqual(True, myObj.canReplace) self.assertEqual(False, myObj.prevWs) self.assertRaises(PpToken.ExceptionCpipTokenIllegalOperation, myObj.setReplace, True)
def test_01(self): """TestPpTokenCount increment.""" myT = PpToken.PpToken('spam', 'identifier') self._ptc.inc(myT, True) self.assertEqual(1, self._ptc.totalAll) self.assertEqual(1, self._ptc.totalAllUnconditional) self.assertEqual(0, self._ptc.totalAllConditional) self.assertEqual(1, self._ptc.tokenCount('identifier', True)) self.assertEqual(1, self._ptc.tokenCount('identifier', False)) self.assertEqual(0, self._ptc.tokenCount('whitespace', True)) self.assertEqual(1, self._ptc.tokenCountNonWs(True)) self.assertEqual(1, self._ptc.tokenCountNonWs(False))
def pprintReplacementList(self, theList): """Pretty prints the replacement list.""" #print i = 0 for aTtt in theList: # PpTokeniser.NAME_ENUM['preprocessing-op-or-punc']), print('%2d: %s,' \ % (i, self.__stringiseToken(aTtt))) i += 1 # TODO: This is horrible we expand the list with None's then pass them to something that thinks they are PpTokens print('As string:') print(PpToken.tokensStr(theList))
def test_shrinkWs_00(self): """PpToken() test shrinkWs() succeeds.""" myObj_00 = PpToken.PpToken(' \n \n ', 'whitespace') self.assertEqual(' \n \n ', myObj_00.t) self.assertEqual('whitespace', myObj_00.tt) self.assertEqual(False, myObj_00.canReplace) self.assertEqual(False, myObj_00.prevWs) self.assertEqual(True, myObj_00.isWs()) myObj_00.shrinkWs() self.assertEqual(myObj_00.SINGLE_SPACE, myObj_00.t) self.assertEqual('whitespace', myObj_00.tt) self.assertEqual(False, myObj_00.canReplace) self.assertEqual(False, myObj_00.prevWs) self.assertEqual(True, myObj_00.isWs())