def test_seek_after_reading_to_end(self): with tempfile.TemporaryDirectory() as tmpdirname: logging.info('created temporary directory "%s"', tmpdirname) filebase = tmpdirname + '/mylog-' sample_lines = [] for f in sorted(SAMPLE_DATA_2): create_file(filebase + f, SAMPLE_DATA_2[f]) sample_lines.extend(SAMPLE_DATA_2[f]) START_TIMESTAMP = get_msec_timestamp(sample_lines[0]) END_TIMESTAMP = get_msec_timestamp(sample_lines[-1]) reader = LogfileReader(filebase) while reader.read() is not None: pass self.assertEqual(END_TIMESTAMP, reader.seek_time(0, 'current')) self.assertEqual(None, reader.read()) self.assertEqual(END_TIMESTAMP, reader.seek_time(0, 'end')) self.assertEqual(None, reader.read()) self.assertEqual(END_TIMESTAMP - 1000, reader.seek_time(-1000, 'current')) self.assertEqual(sample_lines[9], reader.read()) while reader.read() is not None: pass self.assertEqual(END_TIMESTAMP - 1000, reader.seek_time(-1000, 'end')) self.assertEqual(sample_lines[9], reader.read())
def test_tail_false(self): # Don't specify 'tail' and expect there to be no data with tempfile.TemporaryDirectory() as tmpdirname: logging.info('created temporary directory "%s"', tmpdirname) # Create a file slowly, one line at a time target = 'mylogfile' tmpfilename = tmpdirname + '/' + target sample_lines = SAMPLE_DATA.split('\n') threading.Thread(target=create_file, args=(tmpfilename, sample_lines, 0.25)).start() time.sleep(0.05) # let the thread get started # Read, and wait for lines to come reader = LogfileReader(tmpfilename, tail=False) self.assertEqual(None, reader.read())
def test_tail_true(self): # Do the same thing as test_tail_false, but specify tail=True. We should # now get all the lines that are eventually written to the file. with tempfile.TemporaryDirectory() as tmpdirname: logging.info('created temporary directory "%s"', tmpdirname) # Create a file slowly, one line at a time target = 'mylogfile' tmpfilename = tmpdirname + '/' + target sample_lines = SAMPLE_DATA.split('\n') threading.Thread(target=create_file, args=(tmpfilename, sample_lines, 0.25)).start() time.sleep(0.05) # let the thread get started # Read, and wait for lines to come reader = LogfileReader(tmpfilename, tail=True) for line in sample_lines: self.assertEqual(line, reader.read())
class SimNetwork: """Open a network port and feed stored logfile data to it.""" ############################ def __init__(self, port, filebase, instrument): """ ``` port - UDP port on which to write records. filebase - Prefix string to be matched (with a following "*") to fine files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud instrument - Instrument name prefix to add before sendind out on wire ``` """ self.filebase = filebase self.reader = LogfileReader(filebase=filebase, use_timestamps=True) self.slice_n = SliceTransform( fields='1:') # grab 2nd and subsequent fields self.timestamp = TimestampTransform() self.prefix = PrefixTransform(instrument) self.writer = UDPWriter(port=port) self.instrument = instrument self.first_time = True self.quit_flag = False ############################ def run(self, loop=False): """Start reading and writing data. If loop==True, loop when reaching end of input. """ logging.info('Starting %s', self.instrument) try: while not self.quit_flag: record = self.reader.read() # If we don't have a record, we're (probably) at the end of # the file. If it's the first time we've tried reading, it # means we probably didn't get a usable file. Either break out # (if we're not looping, or if we don't have a usable file), # or start reading from the beginning (if we are looping and # have a usable file). if not record: if not loop or self.first_time: break logging.info('Looping instrument %s', self.instrument) self.reader = LogfileReader(filebase=self.filebase, use_timestamps=True) continue # Strip off timestamp and tack on a new one record = self.slice_n.transform(record) record = self.timestamp.transform(record) # Add instrument name back on, and write to specified network record = self.prefix.transform(record) self.writer.write(record) self.first_time = False except (OSError, KeyboardInterrupt): self.quit_flag = True logging.info('Finished %s', self.instrument)
class SimSerial: """Create a virtual serial port and feed stored logfile data to it.""" ############################ def __init__(self, port, prefix=None, timestamp=False, time_format=TIME_FORMAT, filebase=None, eol='\n', baudrate=9600, bytesize=8, parity='N', stopbits=1, timeout=None, xonxoff=False, rtscts=False, write_timeout=None, dsrdtr=False, inter_byte_timeout=None, exclusive=None): """ Simulate a serial port, feeding it data from the specified file. ``` port - Temporary serial port to create and make available for reading records. prefix - If non-empty, prefix name prefix to add timestamp = If True, apply current timestamp to record time_format - What format to use for timestamp ``` """ # We'll create two virtual ports: 'port' and 'port_in'; we will write # to port_in and read the values back out from port self.read_port = port self.write_port = port + '_in' self.prefix = PrefixTransform(prefix) if prefix else None self.timestamp = TimestampTransform() if timestamp else None self.time_format = time_format self.filebase = filebase self.serial_params = None # Complain, but go ahead if read_port or write_port exist. for path in [self.read_port, self.write_port]: if os.path.exists(path): logging.warning('Path %s exists; overwriting!') # Do we have any files we can actually read from? if not glob.glob(filebase + '*'): logging.warning('%s: no files matching "%s*"', prefix, filebase) return # Set up our parameters self.serial_params = {'baudrate': baudrate, 'byteside': bytesize, 'parity': parity, 'stopbits': stopbits, 'timeout': timeout, 'xonxoff': xonxoff, 'rtscts': rtscts, 'write_timeout': write_timeout, 'dsrdtr': dsrdtr, 'inter_byte_timeout': inter_byte_timeout, 'exclusive': exclusive} self.quit = False # Finally, find path to socat executable self.socat_path = None for socat_path in ['/usr/bin/socat', '/usr/local/bin/socat']: if os.path.exists(socat_path) and os.path.isfile(socat_path): self.socat_path = socat_path if not self.socat_path: raise NameError('Executable "socat" not found on path. Please refer ' 'to installation guide to install socat.') ############################ def _run_socat(self): """Internal: run the actual command.""" verbose = '-d' write_port_params = 'pty,link=%s,raw,echo=0' % self.write_port read_port_params = 'pty,link=%s,raw,echo=0' % self.read_port cmd = [self.socat_path, verbose, #verbose, # repeating makes it more verbose read_port_params, write_port_params, ] try: # Run socat process using Popen, checking every second or so whether # it's died (poll() != None) or we've gotten a quit signal. logging.info('Calling: %s', ' '.join(cmd)) socat_process = subprocess.Popen(cmd) while not self.quit and not socat_process.poll(): try: socat_process.wait(1) except subprocess.TimeoutExpired: pass except Exception as e: logging.error('ERROR: socat command: %s', e) # If here, process has terminated, or we've seen self.quit. We # want both to be true: if we've terminated, set self.quit so that # 'run' loop can exit. If self.quit, terminate process. if self.quit: socat_process.kill() else: self.quit = True logging.info('Finished: %s', ' '.join(cmd)) ############################ def run(self, loop=False): # If self.serial_params is None, it means that either read or # write device already exist, so we shouldn't actually run, or # we'll destroy them. if not self.serial_params: return """Create the virtual port with socat and start feeding it records from the designated logfile. If loop==True, loop when reaching end of input.""" self.socat_thread = threading.Thread(target=self._run_socat, daemon=True) self.socat_thread.start() time.sleep(0.2) self.reader = LogfileReader(filebase=self.filebase, use_timestamps=True, time_format=self.time_format) self.slice_n = SliceTransform('1:') # strip off the first field (timestamp) self.writer = TextFileWriter(self.write_port, truncate=True) logging.info('Starting %s: %s', self.read_port, self.filebase) while not self.quit: try: record = self.reader.read() # get the next record logging.debug('SimSerial got: %s', record) # End of input? If loop==True, re-open the logfile from the start if record is None: if not loop: break self.reader = LogfileReader(filebase=self.filebase, use_timestamps=True, time_format=self.time_format) record = self.slice_n.transform(record) # strip the timestamp if not record: continue if self.timestamp: # do we want to add a timestamp? record = self.timestamp.transform(record) if self.prefix: # do we want to add prefix? record = self.prefix.transform(record) logging.debug('SimSerial writing: %s', record) self.writer.write(record) # and write it to the virtual port except (OSError, KeyboardInterrupt): break # If we're here, we got None from our input, and are done. Signal # for run_socat to exit self.quit = True
class SimUDP: """Open a network port and feed stored logfile data to it.""" ############################ def __init__(self, port, prefix=None, timestamp=False, time_format=TIME_FORMAT, filebase=None, eol='\n'): """ ``` port - UDP port on which to write records. prefix - If non-empty, prefix to add timestamp = If True, apply current timestamp to record time_format - What format to use for timestamp filebase - Prefix string to be matched (with a following "*") to find files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud ``` """ self.port = port self.prefix = PrefixTransform(prefix) if prefix else None self.timestamp = TimestampTransform() if timestamp else None self.time_format = time_format self.filebase = filebase # Do we have any files we can actually read from? if not glob.glob(filebase + '*'): logging.warning('No files matching "%s*"', filebase) self.quit_flag = True return self.reader = LogfileReader(filebase=filebase, use_timestamps=True, time_format=self.time_format) self.slice_n = SliceTransform(fields='1:') # strip off timestamp self.writer = UDPWriter(port=port, eol=eol) self.first_time = True self.quit_flag = False ############################ def run(self, loop=False): """Start reading and writing data. If loop==True, loop when reaching end of input. """ logging.info('Starting %s: %s', self.port, self.filebase) try: while not self.quit_flag: record = self.reader.read() # If we don't have a record, we're (probably) at the end of # the file. If it's the first time we've tried reading, it # means we probably didn't get a usable file. Either break out # (if we're not looping, or if we don't have a usable file), # or start reading from the beginning (if we are looping and # have a usable file). if not record: if not loop or self.first_time: break logging.info('Looping instrument %s', self.prefix) self.reader = LogfileReader(filebase=self.filebase, use_timestamps=True) continue # Strip off timestamp record = self.slice_n.transform(record) if not record: continue record = record.strip() if not record: continue if self.timestamp: # do we want to add a timestamp? record = self.timestamp.transform(record) if self.prefix: # do we want to add prefix? record = self.prefix.transform(record) self.writer.write(record) self.first_time = False except (OSError, KeyboardInterrupt): self.quit_flag = True logging.info('Finished %s', self.prefix)
class SimUDP: """Open a network port and feed stored logfile data to it.""" ############################ def __init__(self, port, filebase=None, record_format=None, time_format=TIME_FORMAT, eol='\n'): """ ``` port - UDP port on which to write records. time_format - What format to use for timestamp filebase - Prefix string to be matched (with a following "*") to find files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud record_format If specified, a custom record format to use for extracting timestamp and record. The default is '{timestamp:ti} {record}' ``` """ self.port = port self.time_format = time_format self.filebase = filebase self.record_format = record_format or '{timestamp:ti} {record}' self.compiled_record_format = parse.compile(self.record_format) # Do we have any files we can actually read from? if not glob.glob(filebase + '*'): logging.warning('No files matching "%s*"', filebase) self.quit_flag = True return self.reader = LogfileReader(filebase=filebase, use_timestamps=True, record_format=self.record_format, time_format=self.time_format) self.writer = UDPWriter(port=port, eol=eol) self.first_time = True self.quit_flag = False ############################ def run(self, loop=False): """Start reading and writing data. If loop==True, loop when reaching end of input. """ logging.info('Starting %s: %s', self.port, self.filebase) try: while not self.quit_flag: record = self.reader.read() # If we don't have a record, we're (probably) at the end of # the file. If it's the first time we've tried reading, it # means we probably didn't get a usable file. Either break out # (if we're not looping, or if we don't have a usable file), # or start reading from the beginning (if we are looping and # have a usable file). if not record: if not loop or self.first_time: break logging.info('Looping instrument %s', self.prefix) self.reader = LogfileReader( filebase=self.filebase, record_format=self.record_format, use_timestamps=True) continue # We've now got a record. Try parsing timestamp off it try: parsed_record = self.compiled_record_format.parse( record).named record = parsed_record['record'] # We had a problem parsing. Discard record and try reading next one. except (KeyError, ValueError, AttributeError): logging.warning('Unable to parse record into "%s"', self.record_format) logging.warning('Record: %s', record) continue if not record: continue self.writer.write(record) self.first_time = False except (OSError, KeyboardInterrupt): self.quit_flag = True logging.info('Finished %s', self.prefix)
class SimSerial: """Create a virtual serial port and feed stored logfile data to it.""" ############################ def __init__(self, port, time_format=TIME_FORMAT, filebase=None, record_format=None, eol='\n', baudrate=9600, bytesize=8, parity='N', stopbits=1, timeout=None, xonxoff=False, rtscts=False, write_timeout=None, dsrdtr=False, inter_byte_timeout=None, exclusive=None): """ Simulate a serial port, feeding it data from the specified file. ``` port - Temporary serial port to create and make available for reading records. filebase Possibly wildcarded string specifying files to be opened. record_format If specified, a custom record format to use for extracting timestamp and record. The default is '{timestamp:ti} {record}'. time_format - What format to use for timestamp ``` """ # We'll create two virtual ports: 'port' and 'port_in'; we will write # to port_in and read the values back out from port self.read_port = port self.write_port = port + '_in' self.time_format = time_format self.filebase = filebase self.record_format = record_format or '{timestamp:ti} {record}' self.compiled_record_format = parse.compile(self.record_format) self.eol = eol self.serial_params = None # Complain, but go ahead if read_port or write_port exist. for path in [self.read_port, self.write_port]: if os.path.exists(path): logging.warning('Path %s exists; overwriting!', path) # Do we have any files we can actually read from? if not glob.glob(filebase + '*'): logging.warning('No files matching "%s*"', filebase) return # Set up our parameters self.serial_params = { 'baudrate': baudrate, 'byteside': bytesize, 'parity': parity, 'stopbits': stopbits, 'timeout': timeout, 'xonxoff': xonxoff, 'rtscts': rtscts, 'write_timeout': write_timeout, 'dsrdtr': dsrdtr, 'inter_byte_timeout': inter_byte_timeout, 'exclusive': exclusive } self.quit = False ############################ def run(self, loop=False): # If self.serial_params is None, it means that either read or # write device already exist, so we shouldn't actually run, or # we'll destroy them. if not self.serial_params: return try: # Create simulated serial port (something like /dev/ttys2), and link # it to the port they want to connect to (like /tmp/tty_s330). write_fd, read_fd = pty.openpty() # open the pseudoterminal true_read_port = os.ttyname( read_fd) # this is the true filename of port # Get rid of any previous symlink if it exists, and symlink the new pty try: os.unlink(self.read_port) except FileNotFoundError: pass os.symlink(true_read_port, self.read_port) self.reader = LogfileReader(filebase=self.filebase, use_timestamps=True, record_format=self.record_format, time_format=self.time_format) logging.info('Starting %s: %s', self.read_port, self.filebase) while not self.quit: try: record = self.reader.read() # get the next record logging.debug('SimSerial got: %s', record) # End of input? If loop==True, re-open the logfile from the start if record is None: if not loop: break self.reader = LogfileReader( filebase=self.filebase, use_timestamps=True, record_format=self.record_format, time_format=self.time_format) # We've now got a record. Try parsing timestamp off it try: parsed_record = self.compiled_record_format.parse( record).named record = parsed_record['record'] # We had a problem parsing. Discard record and try reading next one. except (KeyError, ValueError, TypeError, AttributeError): logging.warning('Unable to parse record into "%s"', self.record_format) logging.warning('Record: %s', record) continue if not record: continue logging.debug('SimSerial writing: %s', record) os.write(write_fd, (record + self.eol).encode('utf8')) except (OSError, KeyboardInterrupt): break # If we're here, we got None from our input, and are done. Signal # for run_socat to exit self.quit = True finally: # Get rid of the symlink we've created os.unlink(self.read_port)
class SimSerial: """Create a virtual serial port and feed stored logfile data to it.""" ############################ def __init__(self, port, source_file, use_timestamps=True, baudrate=9600, bytesize=8, parity='N', stopbits=1, timeout=None, xonxoff=False, rtscts=False, write_timeout=None, dsrdtr=False, inter_byte_timeout=None, exclusive=None): """Takes source file, whether to deliver data at rate indicated by timestamps, and the standard parameters that a serial port takes.""" self.source_file = source_file self.use_timestamps = use_timestamps # We'll create two virtual ports: 'port' and 'port_in'; we will write # to port_in and read the values back out from port self.read_port = port self.write_port = port + '_in' self.serial_params = { 'baudrate': baudrate, 'byteside': bytesize, 'parity': parity, 'stopbits': stopbits, 'timeout': timeout, 'xonxoff': xonxoff, 'rtscts': rtscts, 'write_timeout': write_timeout, 'dsrdtr': dsrdtr, 'inter_byte_timeout': inter_byte_timeout, 'exclusive': exclusive } self.quit = False # Finally, check that our needed 'socat' actually exists if not subprocess.run(['which', 'socat'], stdout=subprocess.PIPE).stdout: raise NameError( 'Executable "socat" not found on path. Please refer ' 'to installation guide to install socat.') ############################ def _run_socat(self): """Internal: run the actual command.""" verbose = '-d' write_port_params = 'pty,link=%s,raw,echo=0' % self.write_port read_port_params = 'pty,link=%s,raw,echo=0' % self.read_port cmd = [ '/usr/bin/env', 'socat', verbose, #verbose, # repeating makes it more verbose read_port_params, write_port_params, ] try: # Run socat process using Popen, checking every second or so whether # it's died (poll() != None) or we've gotten a quit signal. logging.info('Calling: %s', ' '.join(cmd)) socat_process = subprocess.Popen(cmd) while not self.quit and not socat_process.poll(): try: socat_process.wait(1) except subprocess.TimeoutExpired: pass except Exception as e: logging.error('ERROR: socat command: %s', e) # If here, process has terminated, or we've seen self.quit. We # want both to be true: if we've terminated, set self.quit so that # 'run' loop can exit. If self.quit, terminate process. if self.quit: socat_process.kill() else: self.quit = True logging.info('Finished: %s', ' '.join(cmd)) ############################ def run(self): """Create the virtual port with socat and start feeding it records from the designated logfile.""" self.socat_thread = threading.Thread(target=self._run_socat) self.socat_thread.start() time.sleep(0.2) self.reader = LogfileReader(filebase=self.source_file, use_timestamps=self.use_timestamps) self.strip = SliceTransform('1:') # strip off the first field) self.writer = TextFileWriter(self.write_port, truncate=True) while not self.quit: record = self.reader.read() # get the next record logging.debug('SimSerial got: %s', record) if record is None: break record = self.strip.transform(record) # strip the timestamp if record: logging.debug('SimSerial writing: %s', record) self.writer.write(record) # and write it to the virtual port # If we're here, we got None from our input, and are done. Signal # for run_socat to exit self.quit = True