def test_000_data_framer_construct_destruct(self): """Test constructing and destructing of DataFramer classes.""" uut2 = data_framer.InterwovenLogFramer("") del uut2 uut3 = data_framer.NewlineFramer() del uut3
def __init__(self, device_name, exception_queue, command_queue, log_queue, transport, call_result_queue, raw_data_queue=None, raw_data_id=0, framer=None, partial_line_timeout=PARTIAL_LINE_TIMEOUT, read_timeout=_READ_TIMEOUT, max_read_bytes=_MAX_READ_BYTES, max_write_bytes=_MAX_WRITE_BYTES): """Initialize TransportProcess with the arguments provided. Args: device_name (str): name of device using this transport exception_queue (Queue): to use for reporting exception traceback message from subprocess command_queue (Queue): to receive commands into log_queue (Queue): to write each log line with host stamp added transport (Transport): to use to receive and send raw data call_result_queue (Queue): to write transport call responses to. raw_data_queue (Queue): to put raw (if applicable, detokenized) data into when enabled. raw_data_id (int): unique identifier for data published by this transport process to the raw_data_queue. framer (DataFramer): to use to frame raw data into partial and complete lines. partial_line_timeout (float): time in seconds to wait before adding partial lines to raw_data_queue and log_queue. read_timeout (float): time to wait in seconds for transport reads. max_read_bytes (int): to attempt to read on each transport read call. max_write_bytes (int): to attempt to write on each transport write call. """ process_name = "{}-Transport{}".format(device_name, raw_data_id) super(TransportProcess, self).__init__(device_name, process_name, exception_queue, command_queue, valid_commands=_ALL_VALID_COMMANDS) self._buffered_unicode = u"" self._framer = framer or data_framer.NewlineFramer() self._log_queue = log_queue self._max_read_bytes = max_read_bytes self._max_write_bytes = max_write_bytes self._partial_line_timeout = partial_line_timeout self._partial_log_time = time.time() self._pending_writes = None self._raw_data_enabled = multiprocessing_utils.get_context().Event() self._call_result_queue = call_result_queue self._raw_data_id = raw_data_id self._raw_data_queue = raw_data_queue self._read_timeout = read_timeout self._transport_open = multiprocessing_utils.get_context().Event() self.transport = transport
def test_040_newline_framer_returns_generator(self): """Test LogResponseIdentifier class accepts all lines.""" uut = data_framer.NewlineFramer() lines = uut.get_lines("", begin=0) self.assertIsInstance( lines, types.GeneratorType, "Expected GeneratorType found {}".format(type(lines)))
def get_data_framers(self, num_transports): """Set up framers used to atomicize the raw output of the device. Deals with interwoven lines as well as tokenized output. Default is to use new line character to separate the lines. Args: num_transports (int): number of declared transports. Returns: list: list of data framers mapped to each transport. """ return [data_framer.NewlineFramer()] * num_transports
def __init__(self, device_name, mp_manager, exception_queue, command_queue, parser, log_path, max_read_bytes=_MAX_READ_BYTES, framer=None): """Initialize LogFilterProcess with the arguments provided. Args: device_name (str): name of device used for process and error messages mp_manager (multiprocessing.Manager): object to use for creating Events exception_queue (Queue): to use for reporting exception traceback message from subprocess command_queue (Queue): to receive commands into parser (Parser): object to use for filtering log lines log_path (str): path and filename to write log messages to max_read_bytes (int): to attempt to read from log file each time. framer (DataFramer): to use to frame log data into partial and complete lines. """ super(LogFilterProcess, self).__init__( device_name, device_name + "-LogFilter", mp_manager, exception_queue, command_queue, valid_commands=_VALID_FILTER_COMMANDS) self._buffered_unicode = u"" self._framer = framer or data_framer.NewlineFramer() self._header_length = HOST_TIMESTAMP_LENGTH + LOG_LINE_HEADER_LENGTH self._max_read_bytes = max_read_bytes self._next_log_path = None self._parser = parser self._log_file = None self._log_filename = os.path.basename(log_path) self._log_directory = os.path.dirname(log_path) self._event_file = None self._event_path = get_event_filename(log_path)
def _parse_events(self, log_path, display_refresh): """Parses log file searching for events depositing results into an event file. Args: log_path (str): Path to log filename containing raw, log event data display_refresh (float): Number of seconds to wait prior to refresh of display Raises: ParserError: if log parser fails. Note: With large log files, this process may take a large amount of time. """ logger.info("Parsing log file {} into event file {}, please wait", log_path, self.event_filename) start_time = time.time() with codecs.open(self.event_filename, "a", encoding="utf-8") as event_file: log_filename = os.path.basename(log_path) with codecs.open( log_path, "r", encoding="utf-8", errors="replace") as log_file: log_file.seek(0, os.SEEK_END) total_bytes = log_file.tell() log_file.seek(0, os.SEEK_SET) process_time = start_time buffered_unicode = u"" framer = data_framer.NewlineFramer() new_header_length = ( log_process.HOST_TIMESTAMP_LENGTH + log_process.LOG_LINE_HEADER_LENGTH) old_header_length = 29 try: while True: log_data = log_file.read(size=4096) if time.time() - process_time > display_refresh: process_time = time.time() bytes_processed = log_file.tell() logger.info("{:.2%} complete - bytes processed: {} of {}", bytes_processed / total_bytes, bytes_processed, total_bytes) if not log_data: break log_lines = buffered_unicode + log_data buffered_len = len(buffered_unicode) buffered_unicode = u"" for log_line in framer.get_lines(log_lines, begin=buffered_len): if log_line[-1] == "\n": if "> GDM-" in log_line: header_length = new_header_length else: header_length = old_header_length self._parser_obj.process_line( event_file, log_line, header_length=header_length, log_filename=log_filename) else: buffered_unicode += log_line except IOError as err: logger.debug("log_parser encountered error: {!r}".format(err)) raise errors.ParserError("Log file processing failed. " "IOError: {!r}".format(err)) logger.info("Parsing log file {} into event file {} finished in {}s", log_path, self.event_filename, time.time() - start_time)