def test_process_parser_analytics_count(self): """ Test that we correctly process parser with count metric """ os.environ['TERM'] = 'dumb' app = Logria(None, False, False) # Fake window size: 10 x 100 app.height = 10 app.width = 100 # Set fake previous render app.last_row = app.height - 3 # simulate the last row we can render to app.current_end = 80 # Simulate the last message rendered # Set fake messages app.messages = [str(x) for x in range(10)] app.parser_index = 0 app.last_index_processed = 0 app.analytics_enabled = True # Set parser, activate app.parser = Parser() app.parser.set_pattern( pattern=r'(\d)', type_='regex', name='Test', example='4', analytics_methods={ 'Item': 'count' } ) # Set analytics method manually app.parser._analytics_map = dict( zip(range(len(app.parser._analytics_methods.keys())), app.parser._analytics_methods.keys())) # Store previous message pointer app.previous_messages = app.messages # Process parser process_parser(app) self.assertEqual( app.messages, ['Item', ' 0: 1', ' 1: 1', ' 2: 1', ' 3: 1', ' 4: 1'])
def test_process_parser_invalid_index(self): """ Test that we correctly process parser with invalid index """ os.environ['TERM'] = 'dumb' app = Logria(None, False, False) # Fake window size: 10 x 100 app.height = 10 app.width = 100 # Set fake previous render app.last_row = app.height - 3 # simulate the last row we can render to app.current_end = 80 # Simulate the last message rendered # Set fake messages app.messages = [f'{x}+{x}+{x}' for x in range(10)] app.parser_index = 3 app.last_index_processed = 0 # Set parser, activate app.parser = Parser() app.parser.set_pattern( pattern='\+', type_='split', name='Test', example='a-a', analytics_methods={ 'Item 1': 'count', 'Item 2': 'count' } ) # Store previous message pointer app.previous_messages = app.messages # Process parser process_parser(app) self.assertEqual(app.messages, [])
def test_process_parser_analytics_average_no_numbers(self): """ Test that we correctly process a parser with average metric but no source numbers """ os.environ['TERM'] = 'dumb' app = Logria(None, False, False) # Fake window size: 10 x 100 app.height = 10 app.width = 100 # Set fake previous render app.last_row = app.height - 3 # simulate the last row we can render to app.current_end = 80 # Simulate the last message rendered # Set fake messages app.messages = [chr(x) for x in range(64, 80)] app.parser_index = 0 app.last_index_processed = 0 app.analytics_enabled = True # Set parser, activate app.parser = Parser() app.parser.set_pattern( pattern=r'(\d)', type_='regex', name='Test', example='4', analytics_methods={ 'Item': 'average' } ) # Set analytics method manually app.parser._analytics_map = dict( zip(range(len(app.parser._analytics_methods.keys())), app.parser._analytics_methods.keys())) # Since we manually construct alaytics, create the the key app.parser.analytics[0] = None self.assertIsNone(app.parser.apply_analytics(0, 'A'))
def test_process_parser_no_analytics(self): """ Test that we correctly process parser with no analytics """ os.environ['TERM'] = 'dumb' app = Logria(None, False, False) # Fake window size: 10 x 100 app.height = 10 app.width = 100 # Set fake previous render app.last_row = app.height - 3 # simulate the last row we can render to app.current_end = 80 # Simulate the last message rendered # Set fake messages app.messages = [str(x) for x in range(10)] app.parser_index = 0 app.last_index_processed = 0 # Set parser, activate app.parser = Parser() app.parser.set_pattern( pattern=r'(\d)', type_='regex', name='Test', example='4', analytics_methods={ 'Item': 'average' } ) # Store previous message pointer app.previous_messages = app.messages # Process parser process_parser(app) self.assertEqual(app.messages, [str(x) for x in range(10)])
def handle_create_parser(self) -> None: temp_parser = Parser() # Render text self.current_end = 0 self.messages = constants.CREATE_PARSER_MESSAGES self.previous_render = None # Force render self.render_text_in_output() # Get type self.activate_prompt() parser_type = None while parser_type not in {'regex', 'split'}: self.activate_prompt() parser_type = self.box.gather().strip() # Handle next step self.messages = [f'Parser type {parser_type}'] self.messages.append(constants.PARSER_SET_NAME) self.previous_render = None # Force render self.render_text_in_output() # Get name self.activate_prompt() parser_name = self.box.gather().strip() # Handle next step self.messages.append(f'Parser name {parser_name}') self.messages.append(constants.PARSER_SET_EXAMPLE) self.previous_render = None # Force render self.render_text_in_output() # Get example self.activate_prompt() parser_example = self.box.gather().strip() # Handle next step self.messages.append(f'Parser example {parser_example}') self.messages.append(constants.PARSER_SET_PATTERN) self.previous_render = None # Force render self.render_text_in_output() # Get pattern self.activate_prompt() parser_pattern = self.box.gather() # Set the parser's data temp_parser.set_pattern(parser_pattern, parser_type, parser_name, parser_example, {}) # Determine the analytics dict parts = temp_parser.parse(parser_example) analytics = {part: 'count' for part in parts} # Set the parser's data with analytics temp_parser.set_pattern(parser_pattern, parser_type, parser_name, parser_example, analytics) self.messages = temp_parser.as_list() self.messages.append(constants.SAVE_CURRENT_PATTERN) self.previous_render = None # Force render self.render_text_in_output() self.activate_prompt() final_res = self.box.gather().strip() if final_res == ':q': return temp_parser.save() self.messages = []
class Logria(): """ Main app class that controls the logical flow of the app """ def __init__(self, stream: InputStream, poll_rate=0.001): # UI Elements initialized to None self.stdscr = None # The entire window self.outwin: curses.window = None # The output window self.command_line: curses.window = None # The command line self.box: Textbox = None # The text box inside the command line # App state self.poll_rate: float = poll_rate # The rate at which we check for new messages self.height: int = None # Window height self.width: int = None # Window width # Store the state of the previous render so we know if we need to refresh self.previous_render: List[str] = None # Pointer to the previous non-parsed message list, which is continuously updated self.previous_messages: List[str] = [] self.exit_val = 0 # If exit_val is -1, the app dies # Message buffers self.stderr_messages: List[str] = [] self.stdout_messages: List[str] = [] self.messages: list = self.stderr_messages # Default to watching stderr # Regex Handler information self.func_handle: callable = None # Regex func that handles filtering self.regex_pattern: str = '' # Current regex pattern # Int array of matches when filtering is active self.matched_rows: List[int] = [] self.last_index_regexed: int = 0 # The last index the filtering function saw # Processor information self.parser: Parser = None # Reference to the current parser self.parser_index: int = 0 # Index for the parser to look at self.parsed_messages: List[dict] = [] # Array of parsed rows self.analytics_enabled: bool = False # Array for statistics messages self.last_index_processed: int = 0 # The last index the parsing function saw # Variables to store the current state of the app self.insert_mode: bool = False # Default to insert mode (like vim) off self.current_status: str = '' # Current status, aka what is in the command line # Determines whether we highlight the match to the user self.highlight_match: bool = True self.last_row: int = 0 # The last row we can render, aka number of lines self.stick_to_bottom: bool = True # Whether we should follow the stream # Whether we should stick to the top and not render new lines self.stick_to_top: bool = False self.manually_controlled_line: bool = False # Whether manual scroll is active self.current_end: bool = 0 # Current last row we have rendered # If we do not have a stream yet, tell the user to set one up if stream is None: self.streams: List[InputStream] = [] else: # Stream list to handle multiple streams self.streams: List[InputStream] = [stream] def build_command_line(self) -> None: """ Creates a textbox object that has insert mode set to the passed value """ if self.command_line: del self.command_line height, width = self.stdscr.getmaxyx() # 1 line, screen width, start 2 from the bottom, 1 char from the side self.command_line = curses.newwin(1, width, height - 2, 1) # Do not block the event loop waiting for input self.command_line.nodelay(True) # Draw rectangle around the command line # upper left: (height - 2, 0), 2 chars up on left edge # lower right: (height, width), bottom right corner of screen rectangle(self.stdscr, height - 3, 0, height - 1, width - 2) self.stdscr.refresh() # Editable text box element self.box = Textbox(self.command_line, insert_mode=self.insert_mode) self.write_to_command_line( self.current_status) # Update current status def setup_streams(self) -> None: """ When launched without a stream, allow the user to define them for us """ # Setup a SessionHandler and get the existing saved sessions session_handler = SessionHandler() # Tell the user what we are doing self.messages.extend(constants.START_MESSAGE) self.messages.extend(session_handler.show_sessions()) self.render_text_in_output() # Dump the existing status self.write_to_command_line('') # Create resolver class to resolve commands resolver = Resolver() # Get user input while True: time.sleep(self.poll_rate) self.activate_prompt() command = self.box.gather().strip() if not command: continue try: command = int(command) session = session_handler.load_session(command) if not session: continue commands = session.get('commands') # Commands need a type for command in commands: if session.get('type') == 'file': self.streams.append(FileInputStream(command)) elif session.get('type') == 'command': self.streams.append(CommandInputStream(command)) except JSONDecodeError as err: self.messages.append( f'Invalid JSON: {err.msg} on line {err.lineno}, char {err.colno}' ) self.render_text_in_output() continue except ValueError: if command == ':config': self.config_mode() return elif command == ':q': self.stop() elif isfile(command): self.streams.append(FileInputStream(command.split('/'))) session_handler.save_session( 'File - ' + command.replace('/', '|'), [command.split('/')], 'file') else: cmd = resolver.resolve_command_as_list(command) self.streams.append(CommandInputStream(cmd)) session_handler.save_session( 'Cmd - ' + command.replace('/', '|'), [cmd], 'command') break # Launch the subprocess for stream in self.streams: stream.poll_rate = self.poll_rate stream.start() # Set status back to what it was self.write_to_command_line(self.current_status) # Render immediately self.previous_render = None # Reset messages self.stderr_messages = [] self.messages = self.stderr_messages def setup_parser(self): """ Setup a parser object in the main runtime """ # Reset the status for new writes self.reset_parser() self.reset_regex_status() # Store previous message pointer if self.messages is self.stderr_messages: self.previous_messages = self.stderr_messages elif self.messages is self.stdout_messages: self.previous_messages = self.stdout_messages # Stick to top to show options self.manually_controlled_line = False self.stick_to_bottom = True self.stick_to_top = False # Overwrite the messages pointer self.messages = Parser().show_patterns() self.previous_render = None self.render_text_in_output() while True: time.sleep(self.poll_rate) self.activate_prompt() command = self.box.gather().strip() if command == 'q': self.reset_parser() return else: try: parser = Parser() parser.load(Parser().patterns()[int(command)]) break except JSONDecodeError as err: self.messages.append( f'Invalid JSON: {err.msg} on line {err.lineno}, char {err.colno}' ) except ValueError: pass # Overwrite a different list this time, and reset it when done self.messages = parser.display_example() self.previous_render = None self.render_text_in_output() while True: time.sleep(self.poll_rate) self.activate_prompt() command = self.box.gather().strip() if command == 'q': self.reset_parser() return else: try: command = int(command) assert command < len(self.messages) self.parser_index = int(command) self.current_status = f'Parsing with {parser.get_name()}, field {parser.get_analytics_for_index(command)}' self.write_to_command_line(self.current_status) break except ValueError: pass except AssertionError: pass # Set parser self.parser = parser # Put pointer back to new array self.messages = self.parsed_messages # Render immediately self.previous_render = None # Stick to bottom again self.last_index_processed = 0 self.stick_to_bottom = True self.stick_to_top = False def reset_parser(self): """ Remove the current parser, if any exists """ if self.func_handle: self.current_status = f'Regex with pattern /{self.regex_pattern}/' else: self.current_status = 'No filter applied' # CLI message, rendered after if self.previous_messages: # Move messages pointer to the previous state if self.previous_messages is self.stderr_messages: self.messages = self.stderr_messages else: self.messages = self.stdout_messages self.previous_messages = [] self.parsed_messages = [] # Dump parsed messages self.parser = None # Dump the parser self.analytics_enabled = False # Disable analytics blocker self.parser_index = 0 # Dump the pattern index self.last_index_processed = 0 # Reset the last searched index self.current_end = 0 # We now do not know where to end self.stick_to_bottom = True # Stay at the bottom for the next render self.write_to_command_line(self.current_status) def process_parser(self): """ Load parsed messages to new array if we have matches # TODO: Same as process_matches """ for index in range(self.last_index_processed, len(self.previous_messages)): if self.analytics_enabled: self.parser.handle_analytics_for_message( self.previous_messages[index]) self.messages = self.parser.analytics_to_list() # For some reason this isn't switching back self.last_index_processed = len(self.previous_messages) else: if self.messages is not self.parsed_messages: self.messages = self.parsed_messages match = self.parser.parse(self.previous_messages[index]) if match: try: self.parsed_messages.append(match[self.parser_index]) except IndexError: # If there was an error parsing, the message did not match the current pattern pass self.last_index_processed = len(self.messages) def determine_render_position(self, messages_pointer: List[str]) -> (int, int): """ Determine the start and end positions for a screen render """ if self.stick_to_top: end = 0 rows = 0 for i in messages_pointer: if messages_pointer is self.messages: # No processing needed for normal messages item = i elif messages_pointer is self.matched_rows: # Grab the matched message item = self.messages[i] # Determine if the message will fit in the window msg_lines = ceil(get_real_length(item) / self.width) rows += msg_lines # If we can fit, increment the last row number if rows < self.last_row and end < len(messages_pointer) - 1: end += 1 else: break self.current_end = end # Save this row so we know where we are # When iterating backwards, we need to end at 0, so we must create a range # object like range(10, -1, -1) to generate a list that ends at 0 # If there are no messages, we want to not iterate later, so we change the # -1 to 0 so that we do not iterate at all return -1 if messages_pointer else 0, end # Early escape elif self.stick_to_bottom: end = len(messages_pointer) - 1 elif self.manually_controlled_line: if len(messages_pointer) < self.last_row: # If have fewer messages than lines, just render it all end = len(messages_pointer) - 1 elif self.current_end < self.last_row: # If the last row we rendered comes before the last row we can render, # use all of the available rows end = self.current_end elif self.current_end < len(messages_pointer): # If we are looking at a valid line, render ends there end = self.current_end else: # If we have over-scrolled, go back if self.current_end > len(messages_pointer): self.current_end = len(messages_pointer) # Since current_end can be zero, we have to use the number of messages end = len(messages_pointer) else: end = len(messages_pointer) self.current_end = end # Save this row so we know where we are # Last index of a list is length - 1 start = max(-1, end - self.last_row - 1) return start, end def render_text_in_output(self) -> None: """ Renders stream content in the output window If filters are inactive, we use `messages`. If they are active, we pull from `matched_rows` We write the whole message, regardless of length, because slicing a string allocates a new string """ # Store a pointer to the buffer of messages if self.func_handle is None: messages_pointer = self.messages else: messages_pointer = self.matched_rows # Determine the start and end position of the render start, end = self.determine_render_position(messages_pointer) # Don't do anything if nothing changed; start at index 0 if self.previous_render == messages_pointer[max(start, 0):end]: return self.previous_render = messages_pointer[max(start, 0):end] self.outwin.erase() current_row = self.last_row # The row we are currently rendering for i in range(end, start, -1): if messages_pointer is self.messages: # No processing needed for normal messages item = messages_pointer[i] elif messages_pointer is self.matched_rows: # Grab the matched message and optionally highlight it messages_idx = self.matched_rows[i] item = self.messages[messages_idx] if self.highlight_match: # Remove all color codes before applying highlighter item = re.sub(constants.ANSI_COLOR_PATTERN, '', item) item = re.sub(self.regex_pattern, f'\u001b[35m{self.regex_pattern}\u001b[0m', item.rstrip()) # Find the correct start position current_row -= ceil(get_real_length(item) / self.width) if current_row < 0: break # Instead of window.addstr, handle colors color_handler.addstr(self.outwin, current_row, 0, item.rstrip()) self.outwin.refresh() def process_matches(self) -> None: """ Process the matches for filtering, should by async but the commented code here does not work # TODO: Fix this method """ # def add_to_list(result: multiprocessing.Queue, messages: list, last_idx_searched: int, func_handle: callable): # """ # Main loop will create this separate process to find matches while the main loop runs # """ # for index, message in range(last_idx_searched, len(messages)): # print(index, message) # if func_handle(message): # result.put(index) # return result # result = multiprocessing.Queue() # proc = multiprocessing.Process(target=add_to_list, args=(result, self.messages, self.last_index_searched, self.func_handle)) # proc.start() # proc.join() # print('done') # self.last_index_searched = len(self.messages) # while not result.empty: # idx = result.get() # print(idx) # self.matched_rows.append(idx) # self.write_to_prompt('in method') # For each message, add its index to the list of matches; this is more efficient than # Storing a second copy of each match for index in range(self.last_index_regexed, len(self.messages)): if self.func_handle(self.messages[index]): self.matched_rows.append(index) self.last_index_regexed = len(self.messages) def write_to_command_line(self, string: str) -> None: """ Writes a message to the command line """ self.reset_command_line() curses.curs_set(1) self.command_line.move(0, 0) self.command_line.addstr(0, 0, string) curses.curs_set(0) def reset_command_line(self) -> None: """ Resets the command line """ self.command_line.move(0, 0) self.command_line.deleteln() curses.curs_set(0) def activate_prompt(self, text='') -> None: """ Activate the prompt so we can edit it text: str, some text to prepend to the command """ self.reset_command_line() if text: self.write_to_command_line(text) curses.curs_set(1) self.box.edit(validator) def handle_regex_command(self, command: str) -> None: """ Handle a regex command """ self.reset_regex_status() self.func_handle = regex_test_generator(command) self.highlight_match = True self.regex_pattern = command # Tell the user what is happening since this is synchronous self.current_status = f'Searching buffer for regex /{self.regex_pattern}/' self.write_to_command_line(self.current_status) # Process any new matched messages to render self.process_matches() # Tell the user we are now filtering self.current_status = f'Regex with pattern /{self.regex_pattern}/' self.write_to_command_line(self.current_status) # Render the text self.render_text_in_output() curses.curs_set(0) def reset_regex_status(self) -> None: """ Reset current regex/filter status to no filter """ if self.parser: self.current_status = f'Parsing with {self.parser.get_name()}, field {self.parser_index}' else: self.current_status = 'No filter applied' # CLI message, rendered after self.previous_render = None # Reset previous render self.func_handle = None # Disable filter self.highlight_match = False # Disable highlighting self.regex_pattern = '' # Clear the current pattern self.matched_rows = [] # Clear out matched rows self.last_index_regexed = 0 # Reset the last searched index self.current_end = 0 # We now do not know where to end self.stick_to_bottom = True # Stay at the bottom for the next render self.write_to_command_line(self.current_status) # Render status def handle_create_session_file(self, session: SessionHandler) -> bool: cmd_resolver = Resolver() # The resolver we use to add commands self.messages.append(constants.SESSION_ADD_FILE) self.previous_render = None # Force render self.render_text_in_output() session.set_type('file') self.activate_prompt() file_path = self.box.gather().strip() file_path = cmd_resolver.resolve_file_as_list(file_path) if isfile('/'.join(file_path)): session.add_command(file_path) self.messages = session.as_list() self.messages.append(constants.SESSION_SHOULD_CONTINUE_FILE) self.previous_render = None # Force render self.render_text_in_output() self.activate_prompt() user_done = self.box.gather().strip() if user_done == ':s': self.messages = [constants.SAVE_CURRENT_SESSION] self.previous_render = None # Force render self.render_text_in_output() self.activate_prompt() filename = self.box.gather().strip() session.save_current_session(filename) return True elif file_path == ':q': self.stop() else: self.messages.append(f'Cannot resolve path: {"/".join(file_path)}') self.previous_render = None # Force render self.render_text_in_output() return False def handle_create_session_command(self, session: SessionHandler) -> bool: cmd_resolver = Resolver() # The resolver we use to add commands self.messages.append(constants.SESSION_ADD_COMMAND) self.previous_render = None # Force render self.render_text_in_output() session.set_type('command') self.activate_prompt() command = self.box.gather().strip() command = cmd_resolver.resolve_command_as_list(command) session.add_command(command) self.messages = session.as_list() self.messages.append(constants.SESSION_SHOULD_CONTINUE_COMMAND) self.previous_render = None # Force render self.render_text_in_output() self.activate_prompt() user_done = self.box.gather().strip() if user_done == ':s': self.messages = [constants.SAVE_CURRENT_SESSION] self.previous_render = None # Force render self.render_text_in_output() self.activate_prompt() filename = self.box.gather().strip() session.save_current_session(filename) return True elif command == ':q': self.stop() return False def handle_create_session(self) -> None: """ Handle the creation of new sessions """ # Render text self.current_end = 0 self.messages = constants.CREATE_SESSION_START_MESSAGES self.previous_render = None # Force render self.render_text_in_output() # Get the user choice choice = None while choice not in {'file', 'command'}: self.activate_prompt() choice = self.box.gather().strip() if choice == ':q': self.stop() break done = False self.messages = [] temp_session = SessionHandler() # The session object we build while not done: if choice == 'file': done = self.handle_create_session_file(temp_session) elif choice == 'command': done = self.handle_create_session_command(temp_session) elif choice == ':q': self.stop() break else: raise ValueError(f'{choice} not one of ("file", "command")') self.setup_streams() def handle_create_parser(self) -> None: temp_parser = Parser() # Render text self.current_end = 0 self.messages = constants.CREATE_PARSER_MESSAGES self.previous_render = None # Force render self.render_text_in_output() # Get type self.activate_prompt() parser_type = None while parser_type not in {'regex', 'split'}: self.activate_prompt() parser_type = self.box.gather().strip() # Handle next step self.messages = [f'Parser type {parser_type}'] self.messages.append(constants.PARSER_SET_NAME) self.previous_render = None # Force render self.render_text_in_output() # Get name self.activate_prompt() parser_name = self.box.gather().strip() # Handle next step self.messages.append(f'Parser name {parser_name}') self.messages.append(constants.PARSER_SET_EXAMPLE) self.previous_render = None # Force render self.render_text_in_output() # Get example self.activate_prompt() parser_example = self.box.gather().strip() # Handle next step self.messages.append(f'Parser example {parser_example}') self.messages.append(constants.PARSER_SET_PATTERN) self.previous_render = None # Force render self.render_text_in_output() # Get pattern self.activate_prompt() parser_pattern = self.box.gather() # Set the parser's data temp_parser.set_pattern(parser_pattern, parser_type, parser_name, parser_example, {}) # Determine the analytics dict parts = temp_parser.parse(parser_example) analytics = {part: 'count' for part in parts} # Set the parser's data with analytics temp_parser.set_pattern(parser_pattern, parser_type, parser_name, parser_example, analytics) self.messages = temp_parser.as_list() self.messages.append(constants.SAVE_CURRENT_PATTERN) self.previous_render = None # Force render self.render_text_in_output() self.activate_prompt() final_res = self.box.gather().strip() if final_res == ':q': return temp_parser.save() self.messages = [] def config_mode(self) -> None: """ Start the configuration setup """ self.current_end = 0 self.messages = constants.CONFIG_START_MESSAGES self.previous_render = None # Force render self.render_text_in_output() choice = None while choice not in {'session', 'parser'}: self.activate_prompt() choice = self.box.gather().strip() if choice == ':q': self.stop() break if choice == 'session': self.handle_create_session() elif choice == 'parser': self.handle_create_parser() def handle_regex_mode(self) -> None: """ Handle when user activates regex mode, including parsing textbox message """ if not self.analytics_enabled: # Disable regex in analytics view # Handle getting input from the command line for regex self.activate_prompt() command = self.box.gather().strip() if command: if command == ':q': self.reset_regex_status() else: self.handle_regex_command(command) else: # If command is an empty string, ignore the input self.reset_regex_status() self.reset_command_line() def handle_command_mode(self) -> None: """ Handle when user activates command mode, including parsing textbox message """ # Handle getting input from the command line for commands self.activate_prompt(':') command = self.box.gather().strip() curses.curs_set(0) if command: if command == ':q': self.stop() elif ':poll' in command: try: command = float(command.replace(':poll', '')) self.poll_rate = command for stream in self.streams: stream.poll_rate = command except ValueError: pass elif ':config' in command: self.config_mode() self.reset_command_line() def start(self) -> None: """ Starts the program """ curses.wrapper(self.main) def stop(self) -> None: """ Die if we send an exit signal of -1 """ for stream in self.streams: stream.exit() self.exit_val = -1 def main(self, stdscr) -> None: """ Main program loop, handles user control and logical flow """ curses.use_default_colors() self.stdscr = stdscr stdscr.keypad(1) height, width = stdscr.getmaxyx() # Get screen size # Save these values self.height = height self.width = width - 1 # Setup Output window output_start_row = 0 # Leave space for top border output_height = height - 3 # Leave space for command line self.last_row = output_height - output_start_row # The last row we can write to # Create the window with these sizes self.outwin = curses.newwin(output_height, width - 1, output_start_row, 0) self.outwin.refresh() # Setup Command line self.build_command_line() # Update the command line status self.reset_regex_status() # Disable cursor: curses.curs_set(0) # Start the main app loop while True: if not self.streams: self.setup_streams() # Update messages from the input stream's queues, track time t_0 = time.perf_counter() for stream in self.streams: while not stream.stderr.empty(): message = stream.stderr.get() self.stderr_messages.append(message) while not stream.stdout.empty(): message = stream.stdout.get() self.stdout_messages.append(message) # Prevent this loop from taking up 100% of the CPU dedicated to the main thread by delaying loops t_1 = time.perf_counter() - t_0 # Don't delay if the queue processing took too long time.sleep(max(0, self.poll_rate - t_1)) try: keypress = self.command_line.getkey() # Get keypress if keypress == '/': self.handle_regex_mode() if keypress == ':': self.handle_command_mode() elif keypress == 'h': self.previous_render = None # Force render if self.func_handle and self.highlight_match: self.highlight_match = False elif self.func_handle and not self.highlight_match: self.highlight_match = True else: self.highlight_match = False elif keypress == 'i': # Toggle insert mode if self.insert_mode: self.insert_mode = False else: self.insert_mode = True self.build_command_line() elif keypress == 's': self.previous_render = None # Force render # Swap stdout and stderr self.reset_parser() self.reset_regex_status() if self.messages is self.stderr_messages: self.messages = self.stdout_messages else: self.messages = self.stderr_messages elif keypress == 'p': # Enable parser if self.parser is not None: self.reset_parser() self.setup_parser() elif keypress == 'a': # Enable analytics engine if self.parser is not None: self.last_index_processed = 0 self.parser.reset_analytics() if self.analytics_enabled: self.current_status = f'Parsing with {self.parser.get_name()}, field {self.parser.get_analytics_for_index(self.parser_index)}' self.parsed_messages = [] self.analytics_enabled = False else: self.analytics_enabled = True self.current_status = f'Parsing with {self.parser.get_name()}, analytics view' elif keypress == 'z': # Tear down parser self.reset_parser() elif keypress == 'KEY_UP': # Scroll up self.manually_controlled_line = True self.stick_to_top = False self.stick_to_bottom = False self.current_end = max(0, self.current_end - 1) self.previous_render = None # Force render elif keypress == 'KEY_DOWN': # Scroll down self.manually_controlled_line = True self.stick_to_top = False self.stick_to_bottom = False if self.matched_rows: self.current_end = min( len(self.matched_rows) - 1, self.current_end + 1) else: self.current_end = min( len(self.messages) - 1, self.current_end + 1) self.previous_render = None # Force render elif keypress == 'KEY_RIGHT': # Stick to bottom self.stick_to_top = False self.stick_to_bottom = True self.manually_controlled_line = False elif keypress == 'KEY_LEFT': # Stick to top self.stick_to_top = True self.stick_to_bottom = False self.manually_controlled_line = False except curses.error: # If we have an active filter, process it, always render if self.exit_val == -1: return if self.parser: self.process_parser( ) # This may block if there are a lot of messages if self.func_handle: self.process_matches( ) # This may block if there are a lot of messages self.render_text_in_output()
def setup_parser(self): """ Setup a parser object in the main runtime """ # Reset the status for new writes self.reset_parser() self.reset_regex_status() # Store previous message pointer if self.messages is self.stderr_messages: self.previous_messages = self.stderr_messages elif self.messages is self.stdout_messages: self.previous_messages = self.stdout_messages # Stick to top to show options self.manually_controlled_line = False self.stick_to_bottom = True self.stick_to_top = False # Overwrite the messages pointer self.messages = Parser().show_patterns() self.previous_render = None self.render_text_in_output() while True: time.sleep(self.poll_rate) self.activate_prompt() command = self.box.gather().strip() if command == 'q': self.reset_parser() return else: try: parser = Parser() parser.load(Parser().patterns()[int(command)]) break except JSONDecodeError as err: self.messages.append( f'Invalid JSON: {err.msg} on line {err.lineno}, char {err.colno}' ) except ValueError: pass # Overwrite a different list this time, and reset it when done self.messages = parser.display_example() self.previous_render = None self.render_text_in_output() while True: time.sleep(self.poll_rate) self.activate_prompt() command = self.box.gather().strip() if command == 'q': self.reset_parser() return else: try: command = int(command) assert command < len(self.messages) self.parser_index = int(command) self.current_status = f'Parsing with {parser.get_name()}, field {parser.get_analytics_for_index(command)}' self.write_to_command_line(self.current_status) break except ValueError: pass except AssertionError: pass # Set parser self.parser = parser # Put pointer back to new array self.messages = self.parsed_messages # Render immediately self.previous_render = None # Stick to bottom again self.last_index_processed = 0 self.stick_to_bottom = True self.stick_to_top = False
def setup_parser(logria: 'Logria'): # type: ignore """ Setup a parser object in the main runtime """ # Reset the status for new writes reset_parser(logria) reset_regex_status(logria) # Store previous message pointer if logria.messages is logria.stderr_messages: logria.previous_messages = logria.stderr_messages elif logria.messages is logria.stdout_messages: logria.previous_messages = logria.stdout_messages # Stick to top to show options logria.manually_controlled_line = False logria.stick_to_bottom = True logria.stick_to_top = False # Only set when there are no pattenrs to display so we can update the user custom_message = '' # Create parser object parser = Parser() # Overwrite the messages pointer logria.messages = parser.show_patterns() logria.redraw() while True: if not parser.show_patterns(): parser = None # type: ignore custom_message = 'No parsers found! Enter :config to build one. Press z to cancel.' break time.sleep(logria.poll_rate) logria.activate_prompt() command = logria.box.gather().strip() if command == ':q': logria.messages = logria.previous_messages parser = None # type: ignore break if ':r ' in command[:3]: items_to_remove = resolve_delete_command(command) sessions = parser.patterns() for item in items_to_remove: if item in sessions: parser.remove(sessions[item]) logria.messages = parser.show_patterns() logria.redraw() continue try: parser.load(parser.patterns()[int(command)]) break except KeyError: continue except JSONDecodeError as err: logria.messages.append( f'Invalid JSON: {err.msg} on line {err.lineno}, char {err.colno}' ) except ValueError: pass # Ensure we didn't exit early already if parser: # Overwrite a different list this time, and reset it when done logria.messages = parser.display_example() logria.previous_render = None logria.render_text_in_output() while True: time.sleep(logria.poll_rate) logria.activate_prompt() command = logria.box.gather().strip() if command == ':q': logria.messages = logria.previous_messages logria.previous_render = None reset_parser(logria) return # no need to do any other work try: command = int(command) if command > len(logria.messages): raise ValueError( f'Invalid index {command} not in range 0..{len(logria.messages)}' ) logria.parser_index = int(command) logria.current_status = f'Parsing with {parser.get_name()}, field {parser.get_analytics_for_index(command)}' logria.write_to_command_line(logria.current_status) break except ValueError: pass except AssertionError: pass # Set parser logria.parser = parser # Put pointer back to new list logria.messages = logria.parsed_messages else: logria.messages = logria.previous_messages reset_parser(logria, custom_message=custom_message) # Render immediately logria.previous_render = None # Stick to bottom again logria.last_index_processed = 0 logria.stick_to_bottom = True logria.stick_to_top = False
def handle_create_parser(logria: 'Logria') -> None: # type: ignore """ Get user input to create a session """ temp_parser = Parser() # Render text logria.current_end = 0 logria.messages = constants.CREATE_PARSER_MESSAGES logria.redraw() # Get type logria.activate_prompt() parser_type: str = '' while parser_type not in {'regex', 'split'}: logria.activate_prompt() parser_type = logria.box.gather().strip() if parser_type == ':q': logria.stop() return # Handle next step logria.messages = [f'Parser type {parser_type}'] logria.messages.append(constants.PARSER_SET_NAME) logria.redraw() # Get name logria.activate_prompt() parser_name = logria.box.gather().strip() if parser_name == ':q': logria.stop() return # Handle next step logria.messages.append(f'Parser name {parser_name}') logria.messages.append(constants.PARSER_SET_EXAMPLE) logria.redraw() # Get example logria.activate_prompt() parser_example = logria.box.gather().strip() if parser_example == ':q': logria.stop() return # Handle next step logria.messages.append(f'Parser example {parser_example}') logria.messages.append(constants.PARSER_SET_PATTERN) logria.redraw() # Get pattern logria.activate_prompt() parser_pattern = logria.box.gather() if parser_pattern == ':q': logria.stop() return # Set the parser's data temp_parser.set_pattern(parser_pattern, parser_type, parser_name, parser_example, {}) # Determine the analytics dict parts = temp_parser.parse(parser_example) analytics = {part: 'count' for part in parts} # Set the parser's data with analytics temp_parser.set_pattern(parser_pattern, parser_type, parser_name, parser_example, analytics) logria.messages = temp_parser.as_list() logria.messages.append(constants.SAVE_CURRENT_PATTERN) logria.redraw() logria.activate_prompt() final_res = logria.box.gather().strip() if final_res == ':q': return temp_parser.save() logria.messages = []