def test_tokensStr_01(self):
"""global function tokensStr() test multiple tokens."""
myTokS = [
PpToken.PpToken('f', 'identifier'),
PpToken.PpToken('g', 'identifier'),
]
self.assertEqual('fg', PpToken.tokensStr(myTokS))
def test_01(self):
"""TestPpTokenCount __iadd__ simple"""
#print
self._ptc_1.inc(PpToken.PpToken('a', 'identifier'), True)
self._ptc_2.inc(PpToken.PpToken('a', 'identifier'), True)
#===============================================================================
# print '1 True:'
# pprint.pprint([x for x in self._ptc_1.tokenTypesAndCounts(True, allPossibleTypes=False)])
# print '1 False:'
# pprint.pprint([x for x in self._ptc_1.tokenTypesAndCounts(False, allPossibleTypes=False)])
# print
# print '2 True:'
# pprint.pprint([x for x in self._ptc_2.tokenTypesAndCounts(True, allPossibleTypes=False)])
# print '2 False:'
# pprint.pprint([x for x in self._ptc_2.tokenTypesAndCounts(False, allPossibleTypes=False)])
#===============================================================================
#
self._ptc_1 += self._ptc_2
#===============================================================================
# print
# print 'Merged True:'
# pprint.pprint([x for x in self._ptc_1.tokenTypesAndCounts(True, allPossibleTypes=False)])
# print 'Merged False:'
# pprint.pprint([x for x in self._ptc_1.tokenTypesAndCounts(False, allPossibleTypes=False)])
#===============================================================================
self.assertEqual([
x for x in self._ptc_1.tokenTypesAndCounts(True,
allPossibleTypes=False)
], [('identifier', 2)])
self.assertEqual([
x for x in self._ptc_1.tokenTypesAndCounts(False,
allPossibleTypes=False)
], [('identifier', 2)])
def testMerge_01(self):
"""PpToken() test merge() type='concat' with ws/identifier."""
myObj_00 = PpToken.PpToken(' ', 'whitespace')
self.assertEqual(' ', myObj_00.t)
self.assertEqual('whitespace', myObj_00.tt)
self.assertEqual(False, myObj_00.canReplace)
self.assertEqual(False, myObj_00.prevWs)
self.assertEqual(True, myObj_00.isWs())
myObj_01 = PpToken.PpToken('f', 'identifier')
self.assertEqual('f', myObj_01.t)
self.assertEqual('identifier', myObj_01.tt)
self.assertEqual(True, myObj_01.canReplace)
self.assertEqual(False, myObj_01.prevWs)
self.assertEqual(False, myObj_01.isWs())
#self.assertRaises(
# PpToken.ExceptionCpipTokenIllegalMerge,
# myObj_00.merge,
# myObj_01)
#self.assertRaises(
# PpToken.ExceptionCpipTokenIllegalMerge,
# myObj_01.merge,
# myObj_00)
myObj_00.merge(myObj_01)
self.assertEqual(' f', myObj_00.t)
self.assertEqual('concat', myObj_00.tt)
self.assertEqual(False, myObj_00.canReplace)
self.assertEqual(False, myObj_00.prevWs)
self.assertEqual(False, myObj_00.isWs())
def defined(self, theTtt, flagInvert, theFileLineCol=None):
"""If the PpToken theTtt is an identifier that is currently defined
then this returns 1 as a PpToken, 0 as a PpToken otherwise.
If the macro exists in the environment its reference count is
incremented.
See: ISO/IEC 9899:1999 (E) 6.10.1.
theFileLineCol is a FileLocation.FileLineCol object."""
# If myTtt is newline then raise as this is #if defined\n
if not theTtt.isIdentifier():
raise ExceptionMacroEnv(
'defined() on non-identifier but: %s' % theTtt)
try:
self._defineMap[theTtt.t].incRefCount(theFileLineCol)
if flagInvert:
return PpToken.PpToken('0', 'pp-number')
return PpToken.PpToken('1', 'pp-number')
except KeyError:
try:
self._ifDefAbsentMacros[theTtt.t].append(theFileLineCol)
except KeyError:
self._ifDefAbsentMacros[theTtt.t] = [theFileLineCol,]
# No macro of this name defined
if flagInvert:
return PpToken.PpToken('1', 'pp-number')
return PpToken.PpToken('0', 'pp-number')
def test_05(self):
"""TestPpTokenEvalConstExpr.test_05(): Integer literals, unsigned/long, short strings."""
myTok = PpToken.PpToken('1ul', 'pp-number')
self.assertEqual('1', myTok.evalConstExpr())
myTok = PpToken.PpToken('1U', 'pp-number')
self.assertEqual('1', myTok.evalConstExpr())
myTok = PpToken.PpToken('', 'pp-number')
self.assertEqual('', myTok.evalConstExpr())
def test_tokensStr_02(self):
"""global function tokensStr() test single token, long form."""
myTokS = [
PpToken.PpToken('f', 'identifier'),
]
self.assertEqual(
#'"f", identifier, True, False, False',
'PpToken(t="f", tt=identifier, line=True, prev=False, ?=False)',
PpToken.tokensStr(myTokS, shortForm=False))
def test_01(self):
"""TestPpTokenEvalConstExpr.test_01(): Integer literals."""
myTok = PpToken.PpToken('', 'pp-number')
self.assertEqual('', myTok.evalConstExpr())
myTok = PpToken.PpToken('1', 'pp-number')
self.assertEqual('1', myTok.evalConstExpr())
myTok = PpToken.PpToken('12', 'pp-number')
self.assertEqual('12', myTok.evalConstExpr())
myTok = PpToken.PpToken('123', 'pp-number')
self.assertEqual('123', myTok.evalConstExpr())
def test_06(self):
"""TestPpTokenEvalConstExpr.test_06(): Integer literals, Hex and unsigned/long."""
myTok = PpToken.PpToken('0xFF', 'pp-number')
self.assertEqual('0xFF', myTok.evalConstExpr())
myTok = PpToken.PpToken('0XAF', 'pp-number')
self.assertEqual('0XAF', myTok.evalConstExpr())
myTok = PpToken.PpToken('0xFFul', 'pp-number')
self.assertEqual('0xFF', myTok.evalConstExpr())
myTok = PpToken.PpToken('0XAFul', 'pp-number')
self.assertEqual('0XAF', myTok.evalConstExpr())
def test_07(self):
"""TestPpTokenEvalConstExpr.test_07(): Integer literals, Octal and unsigned/long."""
myTok = PpToken.PpToken('012', 'pp-number')
self.assertEqual('012', myTok.evalConstExpr())
myTok = PpToken.PpToken('017', 'pp-number')
self.assertEqual('017', myTok.evalConstExpr())
myTok = PpToken.PpToken('017uL', 'pp-number')
self.assertEqual('017', myTok.evalConstExpr())
myTok = PpToken.PpToken('017l', 'pp-number')
self.assertEqual('017', myTok.evalConstExpr())
def test_STDC_FP_CONTRACT_ON_fail_02(self):
"""TestPragmaHandlerSTDC: Test FP_CONTRACT ON no ON."""
myH = PragmaHandler.PragmaHandlerSTDC()
myTokS = [
PpToken.PpToken('STDC', 'identifier'),
PpToken.PpToken(' ', 'whitespace'),
PpToken.PpToken('FP_CONTRACT', 'identifier'),
PpToken.PpToken(' ', 'whitespace'),
PpToken.PpToken('\n', 'whitespace'),
]
self.assertEqual(myH.pragma(myTokS), '')
def test_00(self):
"""TestPpTokenEvalConstExpr.test_00(): Floating point literals."""
myTok = PpToken.PpToken('1.23e4', 'pp-number')
self.assertEqual('1.23e4', myTok.evalConstExpr())
myTok = PpToken.PpToken('1.23e4f', 'pp-number')
self.assertEqual('1.23e4', myTok.evalConstExpr())
myTok = PpToken.PpToken('1.23e4F', 'pp-number')
self.assertEqual('1.23e4', myTok.evalConstExpr())
myTok = PpToken.PpToken('1.23e4l', 'pp-number')
self.assertEqual('1.23e4', myTok.evalConstExpr())
myTok = PpToken.PpToken('1.23e4L', 'pp-number')
self.assertEqual('1.23e4', myTok.evalConstExpr())
def test_10(self):
"""TestPpTokenEvalConstExpr.test_10(): Identifiers."""
myTok = PpToken.PpToken('ABC', 'identifier')
self.assertEqual('0', myTok.evalConstExpr())
myTok = PpToken.PpToken('true', 'identifier')
self.assertEqual('True', myTok.evalConstExpr())
myTok = PpToken.PpToken('false', 'identifier')
self.assertEqual('False', myTok.evalConstExpr())
myTok = PpToken.PpToken('&&', 'preprocessing-op-or-punc')
self.assertEqual('and', myTok.evalConstExpr())
myTok = PpToken.PpToken('||', 'preprocessing-op-or-punc')
self.assertEqual('or', myTok.evalConstExpr())
def test_02(self):
"""TestPpLexerSimpleReplace: Test with "spam\\n" * 1000, #define spam eggs and #define eggs chips."""
myCont = '#define spam eggs\n'
myCont += '#define eggs chips\n'
myCont += 'spam\n' * self.NUM_TESTS
myL = self.simpleLexWithContent(myCont)
myToks, myTime = self.runLex(myL)
#logging.info('Rate: %6.1f tokens/second.' % (len(myToks) / myTime))
self.assertEqual(
[
PpToken.PpToken('\n', 'whitespace'),
PpToken.PpToken('\n', 'whitespace'),
] + self.stringToTokens('chips\n' * self.NUM_TESTS),
myToks,
)
def testInit_01(self):
"""PpToken() ctor good: PpToken('spam', 1)"""
myObj = PpToken.PpToken('spam', 1)
self.assertEqual('spam', myObj.t)
self.assertEqual('identifier', myObj.tt)
self.assertEqual(True, myObj.canReplace)
self.assertEqual(False, myObj.prevWs)
def testWs_00(self):
"""PpToken() test not whitespace with isWs()"""
myObj = PpToken.PpToken('spam', 'identifier')
self.assertEqual('spam', myObj.t)
self.assertEqual('identifier', myObj.tt)
self.assertEqual(True, myObj.canReplace)
self.assertEqual(False, myObj.prevWs)
self.assertEqual(False, myObj.isWs())
def testWs_01(self):
"""PpToken() test is whitespace with isWs()"""
myObj = PpToken.PpToken(' ', 'whitespace')
self.assertEqual(' ', myObj.t)
self.assertEqual('whitespace', myObj.tt)
self.assertEqual(False, myObj.canReplace)
self.assertEqual(False, myObj.prevWs)
self.assertEqual(True, myObj.isWs())
def testInit_02_01(self):
"""PpToken() ctor good: PpToken('<spam>', 0)"""
myObj = PpToken.PpToken('<spam>', 0)
self.assertEqual('<spam>', myObj.t)
self.assertEqual('header-name', myObj.tt)
self.assertEqual(False, myObj.canReplace)
self.assertNotEqual(True, myObj.canReplace)
self.assertEqual(False, myObj.prevWs)
def testTokenGenerator(self):
"""ListAsGenerator: list of PpToken(token, token_type)."""
myList = [
PpToken.PpToken('#', 'preprocessing-op-or-punc'),
PpToken.PpToken('define', 'identifier'),
PpToken.PpToken(' ', 'whitespace'),
PpToken.PpToken('BAR', 'identifier'),
PpToken.PpToken('\n', 'whitespace'),
]
myG = next(ListGen.ListAsGenerator(myList))
self.assertEqual(PpToken.PpToken('#', 'preprocessing-op-or-punc'),
next(myG))
self.assertEqual(PpToken.PpToken('define', 'identifier'), next(myG))
self.assertEqual(PpToken.PpToken(' ', 'whitespace'), next(myG))
self.assertEqual(PpToken.PpToken('BAR', 'identifier'), next(myG))
self.assertEqual(PpToken.PpToken('\n', 'whitespace'), next(myG))
self.assertRaises(StopIteration, next, myG)
def testSetReplace_00(self):
"""PpToken() canReplace success: for ('spam', 'identifier')"""
myObj = PpToken.PpToken('spam', 'identifier')
self.assertEqual('spam', myObj.t)
self.assertEqual('identifier', myObj.tt)
self.assertEqual(True, myObj.canReplace)
self.assertEqual(False, myObj.prevWs)
myObj.canReplace = False
self.assertEqual(False, myObj.canReplace)
def defined(self, theTtt, flagInvert, theFileLineCol=None):
"""If the PpToken theTtt is an identifier that is currently defined
then this returns 1 as a PpToken, 0 as a PpToken otherwise.
If the macro exists in the environment its reference count is
incremented.
*theFileLineCol*
Is a :py:class:`.FileLocation.FileLineCol object`.
See: :title-reference:`ISO/IEC 9899:1999 (E) 6.10.1.`
:param theTtt: The token.
:type theTtt: :py:class:`cpip.core.PpToken.PpToken`
:param flagInvert: Invert the test.
:type flagInvert: ``bool``
:param theFileLineCol: File location.
:type theFileLineCol: ``cpip.core.FileLocation.FileLineCol([str, int, int])``
:returns: :py:class:`cpip.core.PpToken.PpToken` -- A token that is either 0 or 1.
:raises: ``KeyError``
"""
# If myTtt is newline then raise as this is #if defined\n
if not theTtt.isIdentifier():
raise ExceptionMacroEnv('defined() on non-identifier but: %s' %
theTtt)
try:
self._defineMap[theTtt.t].incRefCount(theFileLineCol)
if flagInvert:
return PpToken.PpToken('0', 'pp-number')
return PpToken.PpToken('1', 'pp-number')
except KeyError:
try:
self._ifDefAbsentMacros[theTtt.t].append(theFileLineCol)
except KeyError:
self._ifDefAbsentMacros[theTtt.t] = [
theFileLineCol,
]
# No macro of this name defined
if flagInvert:
return PpToken.PpToken('1', 'pp-number')
return PpToken.PpToken('0', 'pp-number')
def test_shrinkWs_01(self):
"""PpToken() test shrinkWs() fails."""
myObj_00 = PpToken.PpToken('f', 'identifier')
self.assertEqual('f', myObj_00.t)
self.assertEqual('identifier', myObj_00.tt)
self.assertEqual(True, myObj_00.canReplace)
self.assertEqual(False, myObj_00.prevWs)
self.assertEqual(False, myObj_00.isWs())
self.assertRaises(PpToken.ExceptionCpipTokenIllegalOperation,
myObj_00.shrinkWs)
def _reexamineLevel(self, theLevel):
myCont, myStart, myEnd = self.retObjectMacros(theLevel)
myCont += ('%s\n' % myStart) * self.NUM_TESTS
myL = self.simpleLexWithContent(myCont)
myToks, myTime = self.runLex(myL)
myExp = []
for i in range(theLevel):
myExp.append(PpToken.PpToken('\n', 'whitespace'))
myExp += self.stringToTokens(('%s\n' % myEnd) * self.NUM_TESTS)
self.assertEqual(myExp, myToks)
def testMerge_02(self):
"""PpToken() test merge() succeeds with identifier/identifier."""
myObj_00 = PpToken.PpToken('f', 'identifier')
self.assertEqual('f', myObj_00.t)
self.assertEqual('identifier', myObj_00.tt)
self.assertEqual(True, myObj_00.canReplace)
self.assertEqual(False, myObj_00.prevWs)
self.assertEqual(False, myObj_00.isWs())
myObj_01 = PpToken.PpToken('g', 'identifier')
self.assertEqual('g', myObj_01.t)
self.assertEqual('identifier', myObj_01.tt)
self.assertEqual(True, myObj_01.canReplace)
self.assertEqual(False, myObj_01.prevWs)
self.assertEqual(False, myObj_01.isWs())
myObj_00.merge(myObj_01)
self.assertEqual('fg', myObj_00.t)
self.assertEqual('identifier', myObj_00.tt)
self.assertEqual(True, myObj_00.canReplace)
self.assertEqual(False, myObj_00.prevWs)
self.assertEqual(False, myObj_00.isWs())
def testMerge_00(self):
"""PpToken() test merge() succeeds with whitespace."""
myObj_00 = PpToken.PpToken(' ', 'whitespace')
self.assertEqual(' ', myObj_00.t)
self.assertEqual('whitespace', myObj_00.tt)
self.assertEqual(False, myObj_00.canReplace)
self.assertEqual(False, myObj_00.prevWs)
self.assertEqual(True, myObj_00.isWs())
myObj_01 = PpToken.PpToken(' ', 'whitespace')
self.assertEqual(' ', myObj_01.t)
self.assertEqual('whitespace', myObj_01.tt)
self.assertEqual(False, myObj_01.canReplace)
self.assertEqual(False, myObj_01.prevWs)
self.assertEqual(True, myObj_01.isWs())
myObj_00.merge(myObj_01)
self.assertEqual(' ', myObj_00.t)
self.assertEqual('whitespace', myObj_00.tt)
self.assertEqual(False, myObj_00.canReplace)
self.assertEqual(False, myObj_00.prevWs)
self.assertEqual(True, myObj_00.isWs())
def test_isCond_02(self):
"""isCond can be not be set to False."""
myTok = PpToken.PpToken('f', 'identifier')
self.assertEqual(False, myTok.isCond)
myTok.setIsCond()
self.assertEqual(True, myTok.isCond)
try:
myTok.isCond = False
self.fail('No AttributeError raised.')
except AttributeError:
pass
def testPrevWs_00(self):
"""PpToken() can set/clear prevWs for ('spam', 'identifier')"""
myObj = PpToken.PpToken('spam', 'identifier')
self.assertEqual('spam', myObj.t)
self.assertEqual('identifier', myObj.tt)
self.assertEqual(True, myObj.canReplace)
self.assertEqual(False, myObj.prevWs)
myObj.prevWs = True
self.assertEqual(True, myObj.prevWs)
myObj.prevWs = False
self.assertEqual(False, myObj.prevWs)
def testInit_03(self):
"""PpToken() ctor good: with non-identifier types (ints)."""
for i in PpToken.LEX_PPTOKEN_TYPE_ENUM_RANGE:
if PpToken.ENUM_NAME[i] != 'identifier':
myObj = PpToken.PpToken('', i)
self.assertEqual('', myObj.t)
self.assertEqual(PpToken.ENUM_NAME[i], myObj.tt)
self.assertEqual(False, myObj.canReplace)
self.assertNotEqual(True, myObj.canReplace)
self.assertEqual(False, myObj.prevWs)
self.assertRaises(PpToken.ExceptionCpipTokenIllegalOperation,
myObj.setReplace, True)
def test_01(self):
"""TestPpTokenCount increment."""
myT = PpToken.PpToken('spam', 'identifier')
self._ptc.inc(myT, True)
self.assertEqual(1, self._ptc.totalAll)
self.assertEqual(1, self._ptc.totalAllUnconditional)
self.assertEqual(0, self._ptc.totalAllConditional)
self.assertEqual(1, self._ptc.tokenCount('identifier', True))
self.assertEqual(1, self._ptc.tokenCount('identifier', False))
self.assertEqual(0, self._ptc.tokenCount('whitespace', True))
self.assertEqual(1, self._ptc.tokenCountNonWs(True))
self.assertEqual(1, self._ptc.tokenCountNonWs(False))
def pprintReplacementList(self, theList):
"""Pretty prints the replacement list."""
#print
i = 0
for aTtt in theList:
# PpTokeniser.NAME_ENUM['preprocessing-op-or-punc']),
print('%2d: %s,' \
% (i, self.__stringiseToken(aTtt)))
i += 1
# TODO: This is horrible we expand the list with None's then pass them to something that thinks they are PpTokens
print('As string:')
print(PpToken.tokensStr(theList))
def test_shrinkWs_00(self):
"""PpToken() test shrinkWs() succeeds."""
myObj_00 = PpToken.PpToken(' \n \n ', 'whitespace')
self.assertEqual(' \n \n ', myObj_00.t)
self.assertEqual('whitespace', myObj_00.tt)
self.assertEqual(False, myObj_00.canReplace)
self.assertEqual(False, myObj_00.prevWs)
self.assertEqual(True, myObj_00.isWs())
myObj_00.shrinkWs()
self.assertEqual(myObj_00.SINGLE_SPACE, myObj_00.t)
self.assertEqual('whitespace', myObj_00.tt)
self.assertEqual(False, myObj_00.canReplace)
self.assertEqual(False, myObj_00.prevWs)
self.assertEqual(True, myObj_00.isWs())
