def testGetTokenData(self): LexerStateTransitionTest.logger.debug('Testing the matching of a sequence.') # Test data location = (3, 4) start_offet = 0 end_offset = 4 source = 'Jul 11 09:51:54' stream = Stream(source, 'Unit Test Stream') stream.element_buffer = list(source) transition = LexerStateTransition('', start_offet, end_offset, True, None, 'Unit Test State Transition') # Show test data LexerStateTransitionTest.logger.debug('Created stream:\n%s' % (stream.to_pretty_json())) LexerStateTransitionTest.logger.debug('Created state transition:\n%s' % (transition.to_pretty_json())) # Run test data = transition.getTokenData(stream, location) # Show test output LexerStateTransitionTest.logger.debug('Token data found: "%s"' % (data)) LexerStateTransitionTest.logger.debug('Final stream state:\n%s' % (stream.to_pretty_json())) # Verify results assert data == source[:location[0]], 'The data from the accepted buffer is incorrect.' LexerStateTransitionTest.logger.debug('Test succeeded!')
def testGetTokenData(self): LexerStateTransitionTest.logger.debug( 'Testing the matching of a sequence.') # Test data location = (3, 4) start_offet = 0 end_offset = 4 source = 'Jul 11 09:51:54' stream = Stream(source, 'Unit Test Stream') stream.element_buffer = list(source) transition = LexerStateTransition('', start_offet, end_offset, True, None, 'Unit Test State Transition') # Show test data LexerStateTransitionTest.logger.debug('Created stream:\n%s' % (stream.to_pretty_json())) LexerStateTransitionTest.logger.debug('Created state transition:\n%s' % (transition.to_pretty_json())) # Run test data = transition.getTokenData(stream, location) # Show test output LexerStateTransitionTest.logger.debug('Token data found: "%s"' % (data)) LexerStateTransitionTest.logger.debug('Final stream state:\n%s' % (stream.to_pretty_json())) # Verify results assert data == source[:location[ 0]], 'The data from the accepted buffer is incorrect.' LexerStateTransitionTest.logger.debug('Test succeeded!')
def testGetNextElement(self): StreamTest.logger.debug('Test getting next element from the source.') # Test data stream = Stream(self.source, 'Unit Test Stream') number_of_elements = 3 # Show test data StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) # Run test for i in range(number_of_elements): element = stream.getNextElement() buffer = ''.join(stream.element_buffer) # Show test output StreamTest.logger.debug('Fetching element %d.' % (i)) StreamTest.logger.debug('Fetched element: "%s"' % (element)) StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) StreamTest.logger.debug('Fetched element buffer: %s' % (str(stream.element_buffer))) # Verify results assert self.source[ i] == element, 'Stream current element does not equal the source character.' assert stream.current_position == i, 'Stream current position is incorrect.' assert self.source[:i + 1] == buffer, 'Stream buffer does not equal the source sequence.' StreamTest.logger.debug('Test succeeded!')
def testGetNextElement(self): StreamTest.logger.debug('Test getting next element from the source.') # Test data stream = Stream(self.source, 'Unit Test Stream') number_of_elements = 3 # Show test data StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) # Run test for i in range(number_of_elements): element = stream.getNextElement() buffer = ''.join(stream.element_buffer) # Show test output StreamTest.logger.debug('Fetching element %d.' % (i)) StreamTest.logger.debug('Fetched element: "%s"' % (element)) StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) StreamTest.logger.debug('Fetched element buffer: %s' % (str(stream.element_buffer))) # Verify results assert self.source[i] == element, 'Stream current element does not equal the source character.' assert stream.current_position == i, 'Stream current position is incorrect.' assert self.source[:i+1] == buffer, 'Stream buffer does not equal the source sequence.' StreamTest.logger.debug('Test succeeded!')
def testNextState(self): LexerStateTest.logger.debug('Testing the matching of a sequence.') # Test data source = 'Jul 11 09:51:54' stream = Stream(source, 'Unit Test Stream') token = Token('str', Token.ALWAYS_DATA, True, 'Token data') state_b = LexerState([], token, True, 'State B') pattern = '\s+\d{2}\s+' start_offet = 0 end_offset = 4 transition = LexerStateTransition(pattern, start_offet, end_offset, True, state_b, 'A to B Transition') state_a = LexerState([transition], token, False, 'State A') # Show test data LexerStateTest.logger.debug('Created stream:\n%s' % (stream.to_pretty_json())) LexerStateTest.logger.debug('Created state A:\n%s' % (state_a.to_pretty_json())) LexerStateTest.logger.debug('Created state B:\n%s' % (state_b.to_pretty_json())) # Run test next_state = state_a new_token = None while next_state == state_a: current_element = stream.getNextElement() next_state, new_token = state_a.nextState(stream) # Show test output LexerStateTest.logger.debug('Fetching element: %s' % (current_element)) # Show test output LexerStateTest.logger.debug('Token found:\n%s' % (new_token.to_pretty_json())) LexerStateTest.logger.debug('Final stream state:\n%s' % (stream.to_pretty_json())) LexerStateTest.logger.debug('Final lexer state:\n%s' % (next_state.to_pretty_json())) # Verify results token.data = source[:3] assert new_token == token, 'The token found is incorrect.' LexerStateTest.logger.debug('Test succeeded!')
def testAtEos(self): StreamTest.logger.debug('Test when the end of the stream is reached.') # Test data stream = Stream(self.source, 'Unit Test Stream') # Show test data StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) # Run test for i in range(len(self.source)): element = stream.getNextElement() # Show test output StreamTest.logger.debug('Fetching element %d.' % (i)) StreamTest.logger.debug('Fetched element: "%s"' % (element)) StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) StreamTest.logger.debug('Fetched element buffer: %s' % (str(stream.element_buffer))) # Show test output StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) # Verify results assert stream.atEos(), 'Stream buffer is not at the end.' StreamTest.logger.debug('Test succeeded!')
def testFlushElements(self): StreamTest.logger.debug('Test flushing data from the buffer.') # Test data stream = Stream(self.source, 'Unit Test Stream') number_of_elements = 3 # Show test data StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) # Run test for i in range(number_of_elements): element = stream.getNextElement() # Show test output StreamTest.logger.debug('Fetching element %d.' % (i)) StreamTest.logger.debug('Fetched element: "%s"' % (element)) StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) StreamTest.logger.debug('Fetched element buffer: %s' % (str(stream.element_buffer))) stream.flushElements() # Show test output StreamTest.logger.debug('Buffer after flush: %s' % (str(stream.element_buffer))) StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) # Verify results assert len(stream.element_buffer) == 0, 'Stream buffer was not properly emptied.' assert stream.start_position == number_of_elements - 1, 'Stream start position is not correct.' assert stream.current_element == None, 'Stream current element is incorrect.' StreamTest.logger.debug('Test succeeded!')
def testGetAllElements(self): StreamTest.logger.debug('Test getting next element from the source.') # Test data stream = Stream(self.source, 'Unit Test Stream') # Show test data StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) # Run test elements = stream.getAllElements() # Show test output StreamTest.logger.debug('Final stream state:\n%s' % (stream.to_pretty_json())) StreamTest.logger.debug('Fetched element buffer:\n%s' % (str(stream.element_buffer))) # Verify results assert self.source == elements, 'Stream buffer does not equal the source.' StreamTest.logger.debug('Test succeeded!')
def findMatchPattern(self, source, pattern, start, end): # Test data stream = Stream(source, 'Unit Test Stream') stream.element_buffer = list(source) if pattern == Element.EOS: stream.current_position = len(stream.element_buffer) transition = LexerStateTransition(pattern, start, end, True, None, 'Unit Test State Transition') # Show test data LexerStateTransitionTest.logger.debug('Created stream:\n%s' % (stream.to_pretty_json())) LexerStateTransitionTest.logger.debug('Created state transition:\n%s' % (transition.to_pretty_json())) # Run test data = transition.findMatch(stream) new_buffer = stream.element_buffer # Show test output LexerStateTransitionTest.logger.debug('Token data found: "%s"' % (data)) LexerStateTransitionTest.logger.debug('New buffer: %s' % (str(new_buffer))) LexerStateTransitionTest.logger.debug('Final stream state:\n%s' % (stream.to_pretty_json())) return (data, new_buffer)
def testPeek(self): StreamTest.logger.debug('Test getting next element from the source.') # Test data stream = Stream(self.source, 'Unit Test Stream') number_of_elements = 3 # Show test data StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) # Run test for i in range(number_of_elements): element = stream.getNextElement() next_element = stream.peek() # Show test output StreamTest.logger.debug('Fetching element %d.' % (i)) StreamTest.logger.debug('Fetched element: "%s"' % (element)) StreamTest.logger.debug('Peeked at element: "%s"' % (next_element)) StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) StreamTest.logger.debug('Fetched element buffer: %s' % (str(stream.element_buffer))) StreamTest.logger.debug('') # Verify results assert next_element == self.source[i+1], 'Peeked at element does not equal the next element in the source' # Show test output StreamTest.logger.debug('Final stream state:\n%s' % (stream.to_pretty_json())) StreamTest.logger.debug('Fetched element buffer:\n%s' % (str(stream.element_buffer))) StreamTest.logger.debug('Test succeeded!')
def testFlushElementRange(self): StreamTest.logger.debug( 'Test flushing data in a particular range from the buffer.') # Test data stream = Stream(self.source, 'Unit Test Stream') number_of_elements = 9 start = 3 end = 6 # Show test data StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) # Run test for i in range(number_of_elements): element = stream.getNextElement() # Show test output StreamTest.logger.debug('Fetching element %d.' % (i)) StreamTest.logger.debug('Fetched element: "%s"' % (element)) StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) StreamTest.logger.debug('Fetched element buffer: %s' % (str(stream.element_buffer))) stream.flushElementRange(start, end) # Show test output StreamTest.logger.debug('Flushing elements between %d and %d.' % (start, end)) StreamTest.logger.debug('Buffer after flush: %s' % (str(stream.element_buffer))) StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) # Verify results assert len( stream.element_buffer ) == number_of_elements - end + start - 1, 'Stream buffer was not properly emptied.' assert stream.start_position == number_of_elements - 1, 'Stream start position is not correct.' assert stream.current_element == None, 'Stream current element is incorrect.' StreamTest.logger.debug('Test succeeded!')
def testAcceptElements(self): StreamTest.logger.debug('Test accepting data from the buffer.') # Test data stream = Stream(self.source, 'Unit Test Stream') number_of_elements = 3 # Show test data StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) # Run test for i in range(number_of_elements): element = stream.getNextElement() # Show test output StreamTest.logger.debug('Fetching element %d.' % (i)) StreamTest.logger.debug('Fetched element: "%s"' % (element)) StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) StreamTest.logger.debug('Fetched element buffer: %s' % (str(stream.element_buffer))) buffer = stream.acceptElements() # Show test output StreamTest.logger.debug('Accepted buffer: %s' % (str(buffer))) StreamTest.logger.debug('Buffer after accepting elements: %s' % (str(stream.element_buffer))) StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) # Verify results assert len(stream.element_buffer ) == 0, 'Stream buffer was not properly emptied.' assert stream.start_position == number_of_elements - 1, 'Stream start position is not correct.' assert stream.current_element == None, 'Stream current element is incorrect.' assert ''.join( buffer ) == self.source[:number_of_elements], 'Accepted buffer is incorrect.' StreamTest.logger.debug('Test succeeded!')
def testPeek(self): StreamTest.logger.debug('Test getting next element from the source.') # Test data stream = Stream(self.source, 'Unit Test Stream') number_of_elements = 3 # Show test data StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) # Run test for i in range(number_of_elements): element = stream.getNextElement() next_element = stream.peek() # Show test output StreamTest.logger.debug('Fetching element %d.' % (i)) StreamTest.logger.debug('Fetched element: "%s"' % (element)) StreamTest.logger.debug('Peeked at element: "%s"' % (next_element)) StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) StreamTest.logger.debug('Fetched element buffer: %s' % (str(stream.element_buffer))) StreamTest.logger.debug('') # Verify results assert next_element == self.source[ i + 1], 'Peeked at element does not equal the next element in the source' # Show test output StreamTest.logger.debug('Final stream state:\n%s' % (stream.to_pretty_json())) StreamTest.logger.debug('Fetched element buffer:\n%s' % (str(stream.element_buffer))) StreamTest.logger.debug('Test succeeded!')
def testAcceptElementRange(self): StreamTest.logger.debug('Test accepting data in a particular range from the buffer.') # Test data stream = Stream(self.source, 'Unit Test Stream') number_of_elements = 9 start = 3 end = 6 # Show test data StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) # Run test for i in range(number_of_elements): element = stream.getNextElement() # Show test output StreamTest.logger.debug('Fetching element %d.' % (i)) StreamTest.logger.debug('Fetched element: "%s"' % (element)) StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) StreamTest.logger.debug('Fetched element buffer: %s' % (str(stream.element_buffer))) buffer = stream.acceptElementRange(start, end) # Show test output StreamTest.logger.debug('Accepting elements between %d and %d.' % (start, end)) StreamTest.logger.debug('Accepted buffer: %s' % (str(buffer))) StreamTest.logger.debug('Buffer after accepting elements: %s' % (str(stream.element_buffer))) StreamTest.logger.debug('Current stream state:\n%s' % (stream.to_pretty_json())) # Verify results assert len(stream.element_buffer) == number_of_elements - end + start - 1, 'Stream buffer was not properly emptied.' assert stream.start_position == number_of_elements - 1, 'Stream start position is not correct.' assert stream.current_element == None, 'Stream current element is incorrect.' StreamTest.logger.debug('Test succeeded!')