def testGetTokenData(self):
LexerStateTransitionTest.logger.debug('Testing the matching of a sequence.')
# Test data
location = (3, 4)
start_offet = 0
end_offset = 4
source = 'Jul 11 09:51:54'
stream = Stream(source, 'Unit Test Stream')
stream.element_buffer = list(source)
transition = LexerStateTransition('', start_offet, end_offset, True, None, 'Unit Test State Transition')
# Show test data
LexerStateTransitionTest.logger.debug('Created stream:\n%s' % (stream.to_pretty_json()))
LexerStateTransitionTest.logger.debug('Created state transition:\n%s' % (transition.to_pretty_json()))
# Run test
data = transition.getTokenData(stream, location)
# Show test output
LexerStateTransitionTest.logger.debug('Token data found: "%s"' % (data))
LexerStateTransitionTest.logger.debug('Final stream state:\n%s' % (stream.to_pretty_json()))
# Verify results
assert data == source[:location[0]], 'The data from the accepted buffer is incorrect.'
LexerStateTransitionTest.logger.debug('Test succeeded!')
def testGetTokenData(self):
LexerStateTransitionTest.logger.debug(
'Testing the matching of a sequence.')
# Test data
location = (3, 4)
start_offet = 0
end_offset = 4
source = 'Jul 11 09:51:54'
stream = Stream(source, 'Unit Test Stream')
stream.element_buffer = list(source)
transition = LexerStateTransition('', start_offet, end_offset, True,
None, 'Unit Test State Transition')
# Show test data
LexerStateTransitionTest.logger.debug('Created stream:\n%s' %
(stream.to_pretty_json()))
LexerStateTransitionTest.logger.debug('Created state transition:\n%s' %
(transition.to_pretty_json()))
# Run test
data = transition.getTokenData(stream, location)
# Show test output
LexerStateTransitionTest.logger.debug('Token data found: "%s"' %
(data))
LexerStateTransitionTest.logger.debug('Final stream state:\n%s' %
(stream.to_pretty_json()))
# Verify results
assert data == source[:location[
0]], 'The data from the accepted buffer is incorrect.'
LexerStateTransitionTest.logger.debug('Test succeeded!')
def testGetNextElement(self):
StreamTest.logger.debug('Test getting next element from the source.')
# Test data
stream = Stream(self.source, 'Unit Test Stream')
number_of_elements = 3
# Show test data
StreamTest.logger.debug('Current stream state:\n%s' %
(stream.to_pretty_json()))
# Run test
for i in range(number_of_elements):
element = stream.getNextElement()
buffer = ''.join(stream.element_buffer)
# Show test output
StreamTest.logger.debug('Fetching element %d.' % (i))
StreamTest.logger.debug('Fetched element: "%s"' % (element))
StreamTest.logger.debug('Current stream state:\n%s' %
(stream.to_pretty_json()))
StreamTest.logger.debug('Fetched element buffer: %s' %
(str(stream.element_buffer)))
# Verify results
assert self.source[
i] == element, 'Stream current element does not equal the source character.'
assert stream.current_position == i, 'Stream current position is incorrect.'
assert self.source[:i +
1] == buffer, 'Stream buffer does not equal the source sequence.'
StreamTest.logger.debug('Test succeeded!')
def testGetNextElement(self):
StreamTest.logger.debug('Test getting next element from the source.')
# Test data
stream = Stream(self.source, 'Unit Test Stream')
number_of_elements = 3
# Show test data
StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json()))
# Run test
for i in range(number_of_elements):
element = stream.getNextElement()
buffer = ''.join(stream.element_buffer)
# Show test output
StreamTest.logger.debug('Fetching element %d.' % (i))
StreamTest.logger.debug('Fetched element: "%s"' % (element))
StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json()))
StreamTest.logger.debug('Fetched element buffer: %s' % (str(stream.element_buffer)))
# Verify results
assert self.source[i] == element, 'Stream current element does not equal the source character.'
assert stream.current_position == i, 'Stream current position is incorrect.'
assert self.source[:i+1] == buffer, 'Stream buffer does not equal the source sequence.'
StreamTest.logger.debug('Test succeeded!')
def testNextState(self):
LexerStateTest.logger.debug('Testing the matching of a sequence.')
# Test data
source = 'Jul 11 09:51:54'
stream = Stream(source, 'Unit Test Stream')
token = Token('str', Token.ALWAYS_DATA, True, 'Token data')
state_b = LexerState([], token, True, 'State B')
pattern = '\s+\d{2}\s+'
start_offet = 0
end_offset = 4
transition = LexerStateTransition(pattern, start_offet, end_offset,
True, state_b, 'A to B Transition')
state_a = LexerState([transition], token, False, 'State A')
# Show test data
LexerStateTest.logger.debug('Created stream:\n%s' %
(stream.to_pretty_json()))
LexerStateTest.logger.debug('Created state A:\n%s' %
(state_a.to_pretty_json()))
LexerStateTest.logger.debug('Created state B:\n%s' %
(state_b.to_pretty_json()))
# Run test
next_state = state_a
new_token = None
while next_state == state_a:
current_element = stream.getNextElement()
next_state, new_token = state_a.nextState(stream)
# Show test output
LexerStateTest.logger.debug('Fetching element: %s' %
(current_element))
# Show test output
LexerStateTest.logger.debug('Token found:\n%s' %
(new_token.to_pretty_json()))
LexerStateTest.logger.debug('Final stream state:\n%s' %
(stream.to_pretty_json()))
LexerStateTest.logger.debug('Final lexer state:\n%s' %
(next_state.to_pretty_json()))
# Verify results
token.data = source[:3]
assert new_token == token, 'The token found is incorrect.'
LexerStateTest.logger.debug('Test succeeded!')
def testAtEos(self):
StreamTest.logger.debug('Test when the end of the stream is reached.')
# Test data
stream = Stream(self.source, 'Unit Test Stream')
# Show test data
StreamTest.logger.debug('Current stream state:\n%s' %
(stream.to_pretty_json()))
# Run test
for i in range(len(self.source)):
element = stream.getNextElement()
# Show test output
StreamTest.logger.debug('Fetching element %d.' % (i))
StreamTest.logger.debug('Fetched element: "%s"' % (element))
StreamTest.logger.debug('Current stream state:\n%s' %
(stream.to_pretty_json()))
StreamTest.logger.debug('Fetched element buffer: %s' %
(str(stream.element_buffer)))
# Show test output
StreamTest.logger.debug('Current stream state:\n%s' %
(stream.to_pretty_json()))
# Verify results
assert stream.atEos(), 'Stream buffer is not at the end.'
StreamTest.logger.debug('Test succeeded!')
def testFlushElements(self):
StreamTest.logger.debug('Test flushing data from the buffer.')
# Test data
stream = Stream(self.source, 'Unit Test Stream')
number_of_elements = 3
# Show test data
StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json()))
# Run test
for i in range(number_of_elements):
element = stream.getNextElement()
# Show test output
StreamTest.logger.debug('Fetching element %d.' % (i))
StreamTest.logger.debug('Fetched element: "%s"' % (element))
StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json()))
StreamTest.logger.debug('Fetched element buffer: %s' % (str(stream.element_buffer)))
stream.flushElements()
# Show test output
StreamTest.logger.debug('Buffer after flush: %s' % (str(stream.element_buffer)))
StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json()))
# Verify results
assert len(stream.element_buffer) == 0, 'Stream buffer was not properly emptied.'
assert stream.start_position == number_of_elements - 1, 'Stream start position is not correct.'
assert stream.current_element == None, 'Stream current element is incorrect.'
StreamTest.logger.debug('Test succeeded!')
def testGetAllElements(self):
StreamTest.logger.debug('Test getting next element from the source.')
# Test data
stream = Stream(self.source, 'Unit Test Stream')
# Show test data
StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json()))
# Run test
elements = stream.getAllElements()
# Show test output
StreamTest.logger.debug('Final stream state:\n%s' % (stream.to_pretty_json()))
StreamTest.logger.debug('Fetched element buffer:\n%s' % (str(stream.element_buffer)))
# Verify results
assert self.source == elements, 'Stream buffer does not equal the source.'
StreamTest.logger.debug('Test succeeded!')
def findMatchPattern(self, source, pattern, start, end):
# Test data
stream = Stream(source, 'Unit Test Stream')
stream.element_buffer = list(source)
if pattern == Element.EOS:
stream.current_position = len(stream.element_buffer)
transition = LexerStateTransition(pattern, start, end, True, None,
'Unit Test State Transition')
# Show test data
LexerStateTransitionTest.logger.debug('Created stream:\n%s' %
(stream.to_pretty_json()))
LexerStateTransitionTest.logger.debug('Created state transition:\n%s' %
(transition.to_pretty_json()))
# Run test
data = transition.findMatch(stream)
new_buffer = stream.element_buffer
# Show test output
LexerStateTransitionTest.logger.debug('Token data found: "%s"' %
(data))
LexerStateTransitionTest.logger.debug('New buffer: %s' %
(str(new_buffer)))
LexerStateTransitionTest.logger.debug('Final stream state:\n%s' %
(stream.to_pretty_json()))
return (data, new_buffer)
def testPeek(self):
StreamTest.logger.debug('Test getting next element from the source.')
# Test data
stream = Stream(self.source, 'Unit Test Stream')
number_of_elements = 3
# Show test data
StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json()))
# Run test
for i in range(number_of_elements):
element = stream.getNextElement()
next_element = stream.peek()
# Show test output
StreamTest.logger.debug('Fetching element %d.' % (i))
StreamTest.logger.debug('Fetched element: "%s"' % (element))
StreamTest.logger.debug('Peeked at element: "%s"' % (next_element))
StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json()))
StreamTest.logger.debug('Fetched element buffer: %s' % (str(stream.element_buffer)))
StreamTest.logger.debug('')
# Verify results
assert next_element == self.source[i+1], 'Peeked at element does not equal the next element in the source'
# Show test output
StreamTest.logger.debug('Final stream state:\n%s' % (stream.to_pretty_json()))
StreamTest.logger.debug('Fetched element buffer:\n%s' % (str(stream.element_buffer)))
StreamTest.logger.debug('Test succeeded!')
def testAtEos(self):
StreamTest.logger.debug('Test when the end of the stream is reached.')
# Test data
stream = Stream(self.source, 'Unit Test Stream')
# Show test data
StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json()))
# Run test
for i in range(len(self.source)):
element = stream.getNextElement()
# Show test output
StreamTest.logger.debug('Fetching element %d.' % (i))
StreamTest.logger.debug('Fetched element: "%s"' % (element))
StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json()))
StreamTest.logger.debug('Fetched element buffer: %s' % (str(stream.element_buffer)))
# Show test output
StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json()))
# Verify results
assert stream.atEos(), 'Stream buffer is not at the end.'
StreamTest.logger.debug('Test succeeded!')
def testNextState(self):
LexerStateTest.logger.debug('Testing the matching of a sequence.')
# Test data
source = 'Jul 11 09:51:54'
stream = Stream(source, 'Unit Test Stream')
token = Token('str', Token.ALWAYS_DATA, True, 'Token data')
state_b = LexerState([], token, True, 'State B')
pattern = '\s+\d{2}\s+'
start_offet = 0
end_offset = 4
transition = LexerStateTransition(pattern, start_offet, end_offset, True, state_b, 'A to B Transition')
state_a = LexerState([transition], token, False, 'State A')
# Show test data
LexerStateTest.logger.debug('Created stream:\n%s' % (stream.to_pretty_json()))
LexerStateTest.logger.debug('Created state A:\n%s' % (state_a.to_pretty_json()))
LexerStateTest.logger.debug('Created state B:\n%s' % (state_b.to_pretty_json()))
# Run test
next_state = state_a
new_token = None
while next_state == state_a:
current_element = stream.getNextElement()
next_state, new_token = state_a.nextState(stream)
# Show test output
LexerStateTest.logger.debug('Fetching element: %s' % (current_element))
# Show test output
LexerStateTest.logger.debug('Token found:\n%s' % (new_token.to_pretty_json()))
LexerStateTest.logger.debug('Final stream state:\n%s' % (stream.to_pretty_json()))
LexerStateTest.logger.debug('Final lexer state:\n%s' % (next_state.to_pretty_json()))
# Verify results
token.data = source[:3]
assert new_token == token, 'The token found is incorrect.'
LexerStateTest.logger.debug('Test succeeded!')
def findMatchPattern(self, source, pattern, start, end):
# Test data
stream = Stream(source, 'Unit Test Stream')
stream.element_buffer = list(source)
if pattern == Element.EOS:
stream.current_position = len(stream.element_buffer)
transition = LexerStateTransition(pattern, start, end, True, None, 'Unit Test State Transition')
# Show test data
LexerStateTransitionTest.logger.debug('Created stream:\n%s' % (stream.to_pretty_json()))
LexerStateTransitionTest.logger.debug('Created state transition:\n%s' % (transition.to_pretty_json()))
# Run test
data = transition.findMatch(stream)
new_buffer = stream.element_buffer
# Show test output
LexerStateTransitionTest.logger.debug('Token data found: "%s"' % (data))
LexerStateTransitionTest.logger.debug('New buffer: %s' % (str(new_buffer)))
LexerStateTransitionTest.logger.debug('Final stream state:\n%s' % (stream.to_pretty_json()))
return (data, new_buffer)
def testGetAllElements(self):
StreamTest.logger.debug('Test getting next element from the source.')
# Test data
stream = Stream(self.source, 'Unit Test Stream')
# Show test data
StreamTest.logger.debug('Current stream state:\n%s' %
(stream.to_pretty_json()))
# Run test
elements = stream.getAllElements()
# Show test output
StreamTest.logger.debug('Final stream state:\n%s' %
(stream.to_pretty_json()))
StreamTest.logger.debug('Fetched element buffer:\n%s' %
(str(stream.element_buffer)))
# Verify results
assert self.source == elements, 'Stream buffer does not equal the source.'
StreamTest.logger.debug('Test succeeded!')
def testFlushElementRange(self):
StreamTest.logger.debug(
'Test flushing data in a particular range from the buffer.')
# Test data
stream = Stream(self.source, 'Unit Test Stream')
number_of_elements = 9
start = 3
end = 6
# Show test data
StreamTest.logger.debug('Current stream state:\n%s' %
(stream.to_pretty_json()))
# Run test
for i in range(number_of_elements):
element = stream.getNextElement()
# Show test output
StreamTest.logger.debug('Fetching element %d.' % (i))
StreamTest.logger.debug('Fetched element: "%s"' % (element))
StreamTest.logger.debug('Current stream state:\n%s' %
(stream.to_pretty_json()))
StreamTest.logger.debug('Fetched element buffer: %s' %
(str(stream.element_buffer)))
stream.flushElementRange(start, end)
# Show test output
StreamTest.logger.debug('Flushing elements between %d and %d.' %
(start, end))
StreamTest.logger.debug('Buffer after flush: %s' %
(str(stream.element_buffer)))
StreamTest.logger.debug('Current stream state:\n%s' %
(stream.to_pretty_json()))
# Verify results
assert len(
stream.element_buffer
) == number_of_elements - end + start - 1, 'Stream buffer was not properly emptied.'
assert stream.start_position == number_of_elements - 1, 'Stream start position is not correct.'
assert stream.current_element == None, 'Stream current element is incorrect.'
StreamTest.logger.debug('Test succeeded!')
def testAcceptElements(self):
StreamTest.logger.debug('Test accepting data from the buffer.')
# Test data
stream = Stream(self.source, 'Unit Test Stream')
number_of_elements = 3
# Show test data
StreamTest.logger.debug('Current stream state:\n%s' %
(stream.to_pretty_json()))
# Run test
for i in range(number_of_elements):
element = stream.getNextElement()
# Show test output
StreamTest.logger.debug('Fetching element %d.' % (i))
StreamTest.logger.debug('Fetched element: "%s"' % (element))
StreamTest.logger.debug('Current stream state:\n%s' %
(stream.to_pretty_json()))
StreamTest.logger.debug('Fetched element buffer: %s' %
(str(stream.element_buffer)))
buffer = stream.acceptElements()
# Show test output
StreamTest.logger.debug('Accepted buffer: %s' % (str(buffer)))
StreamTest.logger.debug('Buffer after accepting elements: %s' %
(str(stream.element_buffer)))
StreamTest.logger.debug('Current stream state:\n%s' %
(stream.to_pretty_json()))
# Verify results
assert len(stream.element_buffer
) == 0, 'Stream buffer was not properly emptied.'
assert stream.start_position == number_of_elements - 1, 'Stream start position is not correct.'
assert stream.current_element == None, 'Stream current element is incorrect.'
assert ''.join(
buffer
) == self.source[:number_of_elements], 'Accepted buffer is incorrect.'
StreamTest.logger.debug('Test succeeded!')
def testPeek(self):
StreamTest.logger.debug('Test getting next element from the source.')
# Test data
stream = Stream(self.source, 'Unit Test Stream')
number_of_elements = 3
# Show test data
StreamTest.logger.debug('Current stream state:\n%s' %
(stream.to_pretty_json()))
# Run test
for i in range(number_of_elements):
element = stream.getNextElement()
next_element = stream.peek()
# Show test output
StreamTest.logger.debug('Fetching element %d.' % (i))
StreamTest.logger.debug('Fetched element: "%s"' % (element))
StreamTest.logger.debug('Peeked at element: "%s"' % (next_element))
StreamTest.logger.debug('Current stream state:\n%s' %
(stream.to_pretty_json()))
StreamTest.logger.debug('Fetched element buffer: %s' %
(str(stream.element_buffer)))
StreamTest.logger.debug('')
# Verify results
assert next_element == self.source[
i +
1], 'Peeked at element does not equal the next element in the source'
# Show test output
StreamTest.logger.debug('Final stream state:\n%s' %
(stream.to_pretty_json()))
StreamTest.logger.debug('Fetched element buffer:\n%s' %
(str(stream.element_buffer)))
StreamTest.logger.debug('Test succeeded!')
def testAcceptElementRange(self):
StreamTest.logger.debug('Test accepting data in a particular range from the buffer.')
# Test data
stream = Stream(self.source, 'Unit Test Stream')
number_of_elements = 9
start = 3
end = 6
# Show test data
StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json()))
# Run test
for i in range(number_of_elements):
element = stream.getNextElement()
# Show test output
StreamTest.logger.debug('Fetching element %d.' % (i))
StreamTest.logger.debug('Fetched element: "%s"' % (element))
StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json()))
StreamTest.logger.debug('Fetched element buffer: %s' % (str(stream.element_buffer)))
buffer = stream.acceptElementRange(start, end)
# Show test output
StreamTest.logger.debug('Accepting elements between %d and %d.' % (start, end))
StreamTest.logger.debug('Accepted buffer: %s' % (str(buffer)))
StreamTest.logger.debug('Buffer after accepting elements: %s' % (str(stream.element_buffer)))
StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json()))
# Verify results
assert len(stream.element_buffer) == number_of_elements - end + start - 1, 'Stream buffer was not properly emptied.'
assert stream.start_position == number_of_elements - 1, 'Stream start position is not correct.'
assert stream.current_element == None, 'Stream current element is incorrect.'
StreamTest.logger.debug('Test succeeded!')