def _translatePhase_1(self):
"""Performs translation phase one.
Note: We do not (yet) support universal-character-name conversion
so this only does trigraphs."""
logging.debug('ItuToTokens._translatePhase_1(): start.')
# Construct a buffer generator
myBg = BufGen.BufGen(self._mps.genChars())
self._fileLocator.startNewPhase()
try:
i = 0
while 1:
# Note: We need to access the array to inch the marker to the
# current character
if myBg[i] == PpTokeniser.TRIGRAPH_PREFIX:
self._mps.setMarker()
if myBg[i+1] == PpTokeniser.TRIGRAPH_PREFIX \
and myBg[i+2] in PpTokeniser.TRIGRAPH_TABLE:
# Do the trigraph replacement
self._mps.removeSetReplaceClear(
isTerm=True,
theType='trigraph',
theRepl=PpTokeniser.TRIGRAPH_TABLE[myBg[i + 2]],
)
i += PpTokeniser.TRIGRAPH_SIZE
self._fileLocator.incCol(PpTokeniser.TRIGRAPH_SIZE)
else:
self._mps.clearMarker()
i += 1
self._fileLocator.update(myBg[i])
else:
i += 1
self._fileLocator.update(myBg[i])
except IndexError:
pass
logging.debug('ItuToTokens._translatePhase_1(): end.')
def _translatePhase_2(self):
"""Performs translation phase two. This does line continuation markers
Note: We do not (yet) test for accidental UCN creation."""
logging.debug('ItuToTokens._translatePhase_2(): start.')
# Construct a buffer generator
myBg = BufGen.BufGen(self._mps.genChars())
self._fileLocator.startNewPhase()
try:
i = 0
while 1:
# Note: We need to access the array to inch the marker to the
# current character
if myBg[i] == '\\':
self._mps.setMarker()
if myBg[i + 1] == '\n':
# Remove the continuation marker
self._mps.removeMarkedWord(isTerm=True)
i += 2
self._fileLocator.incLine()
else:
self._mps.clearMarker()
i += 1
self._fileLocator.update(myBg[i])
else:
i += 1
self._fileLocator.update(myBg[i])
except IndexError:
pass
logging.debug('ItuToTokens._translatePhase_2(): end.')
def test_04(self):
"""Test simple string buffer slicing out of range."""
myStrGen = StrGen('abcdef')
myBg = BufGen.BufGen(next(myStrGen))
try:
myBg[0:8]
self.fail('IndexError not raised.')
except IndexError:
pass
def test_03(self):
"""Test simple string buffer slicing."""
myStrGen = StrGen('abcdef')
myBg = BufGen.BufGen(next(myStrGen))
# Index into it provoking the generator
self.assertEquals(['a', 'b', 'c'], myBg[0:3])
self.assertEquals("BufGen: ['a', 'b', 'c']", str(myBg))
self.assertEquals(['a', 'c', 'e'], myBg[0:6:2])
self.assertEquals("BufGen: ['a', 'b', 'c', 'd', 'e', 'f']", str(myBg))
def test_02(self):
"""Test string buffer use of len() fails."""
myStrGen = StrGen('abc')
myBg = BufGen.BufGen(next(myStrGen))
try:
len(myBg)
self.fail('TypeError not raised.')
except TypeError:
pass
def _translatePhase_3(self):
"""Performs translation phase three. Replaces comments and decomposes
stream into preprocessing tokens.
:returns: ``NoneType``
:raises: ``IndexError``
"""
logging.debug('ItuToTokens._translatePhase_3(): start.')
# Note this is similar to the code in self.genLexPptokenAndSeqWs()
ofsIdx = 0
myBg = BufGen.BufGen(self._mps.genChars())
self._fileLocator.startNewPhase()
try:
while 1:
# Each pass through the loop we find either:
# - Whitespace
# - A comment that is converted to whitespace
# - A preprocessing token
# Reset the token type
self._cppTokType = None
# Poke the BufGen to inch the marker to the current position
myBg[ofsIdx]
self._mps.setMarker()
sliceLen = self._sliceWhitespace(myBg, ofsIdx) \
or self._sliceLexComment(myBg, ofsIdx) \
or self._sliceLexPptoken(myBg, ofsIdx)
if sliceLen > 0:
# Fix comments to replace them by a comment character
if self._cppTokType in PpTokeniser.COMMENT_TYPES:
# Turn the comment into a single whitespace
myIsTerm = self._cppTokType == PpTokeniser.COMMENT_TYPE_C
self._mps.removeSetReplaceClear(
isTerm=myIsTerm,
theType=self._cppTokType,
theRepl=' ')
else:
myIsTerm = self._cppTokType in ('character-literal',
'string-literal',
'non-whitespace')
self._mps.setWordType(self._cppTokType,
isTerm=myIsTerm)
ofsIdx += sliceLen
else:
break
except IndexError:
pass
# Poke input and report if incomplete
try:
myBg[ofsIdx]
self._diagnostic.partialTokenStream(
'lex.pptoken has unparsed tokens %s' % myBg[ofsIdx:],
self.fileLocator)
except IndexError:
pass
logging.debug('ItuToTokens._translatePhase_3(): end.')
def test_01(self):
"""Tests exceeding available slice size."""
myStrGen = StrGen('abc')
myBg = BufGen.BufGen(next(myStrGen))
for a in myBg.gen():
pass
#print a
self.assertEquals(3, myBg.lenBuf)
self.assertRaises(BufGen.ExceptionBufGen, myBg.slice, 99)
self.assertEquals(3, myBg.lenBuf)
def test_00(self):
"""Test simple string buffer generation."""
myStrGen = StrGen('abc')
myBg = BufGen.BufGen(next(myStrGen))
myGen = myBg.gen()
for _a in myGen:
pass
#print a
self.assertEquals(3, myBg.lenBuf)
self.assertEquals("BufGen: ['a', 'b', 'c']", str(myBg))
self.assertEquals(['a', 'b', 'c'], myBg.slice(3))
self.assertEquals(0, myBg.lenBuf)
self.assertEquals('BufGen: []', str(myBg))
def test_03(self):
"""TestBufGenReplace: Test simple string replacement exceeds limits."""
myStrGen = StrGen('aaaccc')
myBg = BufGen.BufGen(next(myStrGen))
myGen = myBg.gen()
for a in myGen:
pass
#print a
self.assertEquals(6, myBg.lenBuf)
self.assertRaises(BufGen.ExceptionBufGen, myBg.replace, -1, 0, [])
self.assertRaises(BufGen.ExceptionBufGen, myBg.replace, 0, -1, [])
self.assertRaises(BufGen.ExceptionBufGen, myBg.replace, 6, 0, [])
self.assertRaises(BufGen.ExceptionBufGen, myBg.replace, 5, 2, [])
def test_00(self):
"""Test simple string buffer indexing."""
myStrGen = StrGen('abc')
myBg = BufGen.BufGen(next(myStrGen))
# Index into it provoking the generator
self.assertEquals('a', myBg[0])
self.assertEquals("BufGen: ['a']", str(myBg))
self.assertEquals(1, myBg.lenBuf)
self.assertEquals('b', myBg[1])
self.assertEquals("BufGen: ['a', 'b']", str(myBg))
self.assertEquals(2, myBg.lenBuf)
self.assertEquals('c', myBg[2])
self.assertEquals("BufGen: ['a', 'b', 'c']", str(myBg))
self.assertEquals(3, myBg.lenBuf)
def test_01(self):
"""TestBufGenReplace: Test simple string replacement "aaabbbccc" - > "aaaccc"."""
myStrGen = StrGen('aaabbbccc')
myBg = BufGen.BufGen(next(myStrGen))
myGen = myBg.gen()
for a in myGen:
pass
#print a
self.assertEquals(9, myBg.lenBuf)
# Now replace 'bbb' with nothing
myBg.replace(3, 3, [])
# Inspect
self.assertEquals(6, myBg.lenBuf)
self.assertEquals(['a', 'a', 'a'], myBg.slice(3))
self.assertEquals(3, myBg.lenBuf)
self.assertEquals(['c', 'c', 'c'], myBg.slice(3))
self.assertEquals(0, myBg.lenBuf)
def test_02(self):
"""TestBufGenReplace: Test simple string replacement "aaaccc" - > "aaaBBBccc"."""
myStrGen = StrGen('aaaccc')
myBg = BufGen.BufGen(next(myStrGen))
myGen = myBg.gen()
for a in myGen:
pass
#print a
self.assertEquals(6, myBg.lenBuf)
# Now insert 'BBB'
myBg.replace(3, 0, ['B', 'B', 'B'])
# Inspect
self.assertEquals(9, myBg.lenBuf)
self.assertEquals(['a', 'a', 'a'], myBg.slice(3))
self.assertEquals(6, myBg.lenBuf)
self.assertEquals(['B', 'B', 'B'], myBg.slice(3))
self.assertEquals(3, myBg.lenBuf)
self.assertEquals(['c', 'c', 'c'], myBg.slice(3))
self.assertEquals(0, myBg.lenBuf)
def test_01(self):
"""Test string buffer indexing out of range."""
myStrGen = StrGen('abc')
myBg = BufGen.BufGen(next(myStrGen))
try:
myBg[-1]
self.fail('IndexError not raised.')
except IndexError:
pass
try:
myBg[4]
self.fail('IndexError not raised.')
except IndexError:
pass
self.assertEquals("BufGen: ['a', 'b', 'c']", str(myBg))
self.assertEquals(3, myBg.lenBuf)
try:
myBg[4]
self.fail('IndexError not raised.')
except IndexError:
pass
self.assertEquals("BufGen: ['a', 'b', 'c']", str(myBg))
self.assertEquals(3, myBg.lenBuf)