def test_prefix(self): prefix_1 = PrefixTransform('p1') prefix_2 = PrefixTransform('p2') f1_name = self.tmpdirname + '/f1' f2_name = self.tmpdirname + '/f2' writer = ComposedWriter( transforms=[prefix_1, prefix_2], writers=[TextFileWriter(f1_name), TextFileWriter(f2_name)]) f1 = open(f1_name, 'r') f2 = open(f2_name, 'r') for line in SAMPLE_DATA: writer.write(line) time.sleep(0.1) f1_line = f1.readline().rstrip() f2_line = f2.readline().rstrip() logging.info('wrote: "%s", f1: "%s", f2: "%s"', line, f1_line, f2_line) self.assertEqual('p2 p1 ' + line, f1_line) self.assertEqual('p2 p1 ' + line, f2_line)
def test_read_all_write_one(self): readers = [] for tmpfilename in self.tmpfilenames: readers.append(TextFileReader(tmpfilename, interval=0.2)) transforms = [PrefixTransform('prefix_1'), PrefixTransform('prefix_2')] outfilename = self.tmpdirname + '/f_out' writers = [TextFileWriter(outfilename)] listener = Listener(readers, transforms, writers) listener.run() out_lines = [] with open(outfilename, 'r') as f: for line in f.readlines(): out_lines.append(line.rstrip()) out_lines.sort() source_lines = [] for f in SAMPLE_DATA: source_lines.extend( ['prefix_2 prefix_1 ' + f for f in SAMPLE_DATA[f]]) source_lines.sort() logging.debug('out: %s, source: %s', out_lines, source_lines) self.assertEqual(out_lines, source_lines)
def __init__(self, port, prefix=None, timestamp=False, time_format=TIME_FORMAT, filebase=None, eol='\n'): """ ``` port - UDP port on which to write records. prefix - If non-empty, prefix to add timestamp = If True, apply current timestamp to record time_format - What format to use for timestamp filebase - Prefix string to be matched (with a following "*") to find files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud ``` """ self.port = port self.prefix = PrefixTransform(prefix) if prefix else None self.timestamp = TimestampTransform() if timestamp else None self.time_format = time_format self.filebase = filebase # Do we have any files we can actually read from? if not glob.glob(filebase + '*'): logging.warning('No files matching "%s*"', filebase) self.quit_flag = True return self.reader = LogfileReader(filebase=filebase, use_timestamps=True, time_format=self.time_format) self.slice_n = SliceTransform(fields='1:') # strip off timestamp self.writer = UDPWriter(port=port, eol=eol) self.first_time = True self.quit_flag = False
def test_all_files(self): # Use TextFileReader's 'interval' flag to make sure we interleave # reads the way we expect. Also make sure transforms get applied # in proper order. readers = [] for tmpfilename in self.tmpfilenames: readers.append(TextFileReader(tmpfilename, interval=0.2)) #readers.append(TextFileReader()) # read from stdin prefix_1 = PrefixTransform('prefix_1') prefix_2 = PrefixTransform('prefix_2') reader = ComposedReader(readers, [prefix_1, prefix_2]) # Clunkly quick way of slicing lines i = 0 expected_lines = [] while True: next_lines = [] for f in sorted(SAMPLE_DATA): if i < len(SAMPLE_DATA[f]): line = 'prefix_2 prefix_1 ' + SAMPLE_DATA[f][i] next_lines.append(line) if next_lines: expected_lines.append(next_lines) i += 1 else: break logging.debug('Expected lines %s', expected_lines) # Next line from each of the files can come in arbitrary order, # but within the file, lines should arrive in order, and we # should receive first line from each file before we receive # next line from any of them. while expected_lines: next_lines = expected_lines.pop(0) while next_lines: record = reader.read() logging.info('read: %s; expected one of: %s', record, next_lines) self.assertTrue(record in next_lines) if record in next_lines: next_lines.remove(record) self.assertEqual(None, reader.read())
def test_default(self): transform = PrefixTransform('prefix') self.assertIsNone(transform.transform(None)) self.assertEqual(transform.transform('foo'), 'prefix foo') transform = PrefixTransform('prefix', sep='\t') self.assertEqual(transform.transform('foo'), 'prefix\tfoo')
def __init__(self, port, filebase, instrument): """ ``` port - UDP port on which to write records. filebase - Prefix string to be matched (with a following "*") to fine files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud instrument - Instrument name prefix to add before sendind out on wire ``` """ self.filebase = filebase self.reader = LogfileReader(filebase=filebase, use_timestamps=True) self.slice_n = SliceTransform( fields='1:') # grab 2nd and subsequent fields self.timestamp = TimestampTransform() self.prefix = PrefixTransform(instrument) self.writer = UDPWriter(port=port) self.instrument = instrument self.first_time = True self.quit_flag = False
def test_check_format(self): f1_name = self.tmpdirname + '/f1' f2_name = self.tmpdirname + '/f2' # This should be okay ComposedWriter( transforms=[PrefixTransform('prefix')], writers=[TextFileWriter(f1_name), TextFileWriter(f2_name)], check_format=True) # Should raise an error if formats are not compatible with self.assertLogs(logging.getLogger(), logging.ERROR): with self.assertRaises(ValueError): ComposedWriter( transforms=[ParseNMEATransform()], writers=[TextFileWriter(f1_name), TextFileWriter(f2_name)], check_format=True)
class SimNetwork: """Open a network port and feed stored logfile data to it.""" ############################ def __init__(self, port, filebase, instrument): """ ``` port - UDP port on which to write records. filebase - Prefix string to be matched (with a following "*") to fine files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud instrument - Instrument name prefix to add before sendind out on wire ``` """ self.filebase = filebase self.reader = LogfileReader(filebase=filebase, use_timestamps=True) self.slice_n = SliceTransform( fields='1:') # grab 2nd and subsequent fields self.timestamp = TimestampTransform() self.prefix = PrefixTransform(instrument) self.writer = UDPWriter(port=port) self.instrument = instrument self.first_time = True self.quit_flag = False ############################ def run(self, loop=False): """Start reading and writing data. If loop==True, loop when reaching end of input. """ logging.info('Starting %s', self.instrument) try: while not self.quit_flag: record = self.reader.read() # If we don't have a record, we're (probably) at the end of # the file. If it's the first time we've tried reading, it # means we probably didn't get a usable file. Either break out # (if we're not looping, or if we don't have a usable file), # or start reading from the beginning (if we are looping and # have a usable file). if not record: if not loop or self.first_time: break logging.info('Looping instrument %s', self.instrument) self.reader = LogfileReader(filebase=self.filebase, use_timestamps=True) continue # Strip off timestamp and tack on a new one record = self.slice_n.transform(record) record = self.timestamp.transform(record) # Add instrument name back on, and write to specified network record = self.prefix.transform(record) self.writer.write(record) self.first_time = False except (OSError, KeyboardInterrupt): self.quit_flag = True logging.info('Finished %s', self.instrument)
def __init__(self, port, prefix=None, timestamp=False, time_format=TIME_FORMAT, filebase=None, eol='\n', baudrate=9600, bytesize=8, parity='N', stopbits=1, timeout=None, xonxoff=False, rtscts=False, write_timeout=None, dsrdtr=False, inter_byte_timeout=None, exclusive=None): """ Simulate a serial port, feeding it data from the specified file. ``` port - Temporary serial port to create and make available for reading records. prefix - If non-empty, prefix name prefix to add timestamp = If True, apply current timestamp to record time_format - What format to use for timestamp ``` """ # We'll create two virtual ports: 'port' and 'port_in'; we will write # to port_in and read the values back out from port self.read_port = port self.write_port = port + '_in' self.prefix = PrefixTransform(prefix) if prefix else None self.timestamp = TimestampTransform() if timestamp else None self.time_format = time_format self.filebase = filebase self.serial_params = None # Complain, but go ahead if read_port or write_port exist. for path in [self.read_port, self.write_port]: if os.path.exists(path): logging.warning('Path %s exists; overwriting!') # Do we have any files we can actually read from? if not glob.glob(filebase + '*'): logging.warning('%s: no files matching "%s*"', prefix, filebase) return # Set up our parameters self.serial_params = {'baudrate': baudrate, 'byteside': bytesize, 'parity': parity, 'stopbits': stopbits, 'timeout': timeout, 'xonxoff': xonxoff, 'rtscts': rtscts, 'write_timeout': write_timeout, 'dsrdtr': dsrdtr, 'inter_byte_timeout': inter_byte_timeout, 'exclusive': exclusive} self.quit = False # Finally, find path to socat executable self.socat_path = None for socat_path in ['/usr/bin/socat', '/usr/local/bin/socat']: if os.path.exists(socat_path) and os.path.isfile(socat_path): self.socat_path = socat_path if not self.socat_path: raise NameError('Executable "socat" not found on path. Please refer ' 'to installation guide to install socat.')
class SimSerial: """Create a virtual serial port and feed stored logfile data to it.""" ############################ def __init__(self, port, prefix=None, timestamp=False, time_format=TIME_FORMAT, filebase=None, eol='\n', baudrate=9600, bytesize=8, parity='N', stopbits=1, timeout=None, xonxoff=False, rtscts=False, write_timeout=None, dsrdtr=False, inter_byte_timeout=None, exclusive=None): """ Simulate a serial port, feeding it data from the specified file. ``` port - Temporary serial port to create and make available for reading records. prefix - If non-empty, prefix name prefix to add timestamp = If True, apply current timestamp to record time_format - What format to use for timestamp ``` """ # We'll create two virtual ports: 'port' and 'port_in'; we will write # to port_in and read the values back out from port self.read_port = port self.write_port = port + '_in' self.prefix = PrefixTransform(prefix) if prefix else None self.timestamp = TimestampTransform() if timestamp else None self.time_format = time_format self.filebase = filebase self.serial_params = None # Complain, but go ahead if read_port or write_port exist. for path in [self.read_port, self.write_port]: if os.path.exists(path): logging.warning('Path %s exists; overwriting!') # Do we have any files we can actually read from? if not glob.glob(filebase + '*'): logging.warning('%s: no files matching "%s*"', prefix, filebase) return # Set up our parameters self.serial_params = {'baudrate': baudrate, 'byteside': bytesize, 'parity': parity, 'stopbits': stopbits, 'timeout': timeout, 'xonxoff': xonxoff, 'rtscts': rtscts, 'write_timeout': write_timeout, 'dsrdtr': dsrdtr, 'inter_byte_timeout': inter_byte_timeout, 'exclusive': exclusive} self.quit = False # Finally, find path to socat executable self.socat_path = None for socat_path in ['/usr/bin/socat', '/usr/local/bin/socat']: if os.path.exists(socat_path) and os.path.isfile(socat_path): self.socat_path = socat_path if not self.socat_path: raise NameError('Executable "socat" not found on path. Please refer ' 'to installation guide to install socat.') ############################ def _run_socat(self): """Internal: run the actual command.""" verbose = '-d' write_port_params = 'pty,link=%s,raw,echo=0' % self.write_port read_port_params = 'pty,link=%s,raw,echo=0' % self.read_port cmd = [self.socat_path, verbose, #verbose, # repeating makes it more verbose read_port_params, write_port_params, ] try: # Run socat process using Popen, checking every second or so whether # it's died (poll() != None) or we've gotten a quit signal. logging.info('Calling: %s', ' '.join(cmd)) socat_process = subprocess.Popen(cmd) while not self.quit and not socat_process.poll(): try: socat_process.wait(1) except subprocess.TimeoutExpired: pass except Exception as e: logging.error('ERROR: socat command: %s', e) # If here, process has terminated, or we've seen self.quit. We # want both to be true: if we've terminated, set self.quit so that # 'run' loop can exit. If self.quit, terminate process. if self.quit: socat_process.kill() else: self.quit = True logging.info('Finished: %s', ' '.join(cmd)) ############################ def run(self, loop=False): # If self.serial_params is None, it means that either read or # write device already exist, so we shouldn't actually run, or # we'll destroy them. if not self.serial_params: return """Create the virtual port with socat and start feeding it records from the designated logfile. If loop==True, loop when reaching end of input.""" self.socat_thread = threading.Thread(target=self._run_socat, daemon=True) self.socat_thread.start() time.sleep(0.2) self.reader = LogfileReader(filebase=self.filebase, use_timestamps=True, time_format=self.time_format) self.slice_n = SliceTransform('1:') # strip off the first field (timestamp) self.writer = TextFileWriter(self.write_port, truncate=True) logging.info('Starting %s: %s', self.read_port, self.filebase) while not self.quit: try: record = self.reader.read() # get the next record logging.debug('SimSerial got: %s', record) # End of input? If loop==True, re-open the logfile from the start if record is None: if not loop: break self.reader = LogfileReader(filebase=self.filebase, use_timestamps=True, time_format=self.time_format) record = self.slice_n.transform(record) # strip the timestamp if not record: continue if self.timestamp: # do we want to add a timestamp? record = self.timestamp.transform(record) if self.prefix: # do we want to add prefix? record = self.prefix.transform(record) logging.debug('SimSerial writing: %s', record) self.writer.write(record) # and write it to the virtual port except (OSError, KeyboardInterrupt): break # If we're here, we got None from our input, and are done. Signal # for run_socat to exit self.quit = True
class SimUDP: """Open a network port and feed stored logfile data to it.""" ############################ def __init__(self, port, prefix=None, timestamp=False, time_format=TIME_FORMAT, filebase=None, eol='\n'): """ ``` port - UDP port on which to write records. prefix - If non-empty, prefix to add timestamp = If True, apply current timestamp to record time_format - What format to use for timestamp filebase - Prefix string to be matched (with a following "*") to find files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud ``` """ self.port = port self.prefix = PrefixTransform(prefix) if prefix else None self.timestamp = TimestampTransform() if timestamp else None self.time_format = time_format self.filebase = filebase # Do we have any files we can actually read from? if not glob.glob(filebase + '*'): logging.warning('No files matching "%s*"', filebase) self.quit_flag = True return self.reader = LogfileReader(filebase=filebase, use_timestamps=True, time_format=self.time_format) self.slice_n = SliceTransform(fields='1:') # strip off timestamp self.writer = UDPWriter(port=port, eol=eol) self.first_time = True self.quit_flag = False ############################ def run(self, loop=False): """Start reading and writing data. If loop==True, loop when reaching end of input. """ logging.info('Starting %s: %s', self.port, self.filebase) try: while not self.quit_flag: record = self.reader.read() # If we don't have a record, we're (probably) at the end of # the file. If it's the first time we've tried reading, it # means we probably didn't get a usable file. Either break out # (if we're not looping, or if we don't have a usable file), # or start reading from the beginning (if we are looping and # have a usable file). if not record: if not loop or self.first_time: break logging.info('Looping instrument %s', self.prefix) self.reader = LogfileReader(filebase=self.filebase, use_timestamps=True) continue # Strip off timestamp record = self.slice_n.transform(record) if not record: continue record = record.strip() if not record: continue if self.timestamp: # do we want to add a timestamp? record = self.timestamp.transform(record) if self.prefix: # do we want to add prefix? record = self.prefix.transform(record) self.writer.write(record) self.first_time = False except (OSError, KeyboardInterrupt): self.quit_flag = True logging.info('Finished %s', self.prefix)
action='count', help='Increase output verbosity') args = parser.parse_args() LOGGING_FORMAT = '%(asctime)-15s %(message)s' logging.basicConfig(format=LOGGING_FORMAT) LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} args.verbosity = min(args.verbosity, max(LOG_LEVELS)) logging.getLogger().setLevel(LOG_LEVELS[args.verbosity]) reader = TextFileReader(file_spec=args.read, tail=args.tail) writer = TextFileWriter(filename=args.write) if args.prefix: prefix_transform = PrefixTransform(args.prefix) if args.timestamp: timestamp_transform = TimestampTransform() while True: record = reader.read() now = time.time() logging.info('Got record: %s', record) if record is None: if not args.tail: break else: if args.timestamp:
if new_args.serial: kwargs = {} for pair in new_args.serial.split(','): (key, value) = pair.split('=') kwargs[key] = value readers.append(SerialReader(**kwargs)) ########################## # Transforms if new_args.slice: transforms.append(SliceTransform(new_args.slice, all_args.slice_separator)) if new_args.timestamp: transforms.append(TimestampTransform(time_format=all_args.time_format)) if new_args.prefix: transforms.append(PrefixTransform(new_args.prefix)) if new_args.extract: transforms.append(ExtractFieldTransform(new_args.extract)) if new_args.regex_filter: transforms.append(RegexFilterTransform(new_args.regex_filter)) if new_args.qc_filter: transforms.append(QCFilterTransform(new_args.qc_filter)) if new_args.parse_nmea: transforms.append( ParseNMEATransform( message_path=all_args.parse_nmea_message_path, sensor_path=all_args.parse_nmea_sensor_path, sensor_model_path=all_args.parse_nmea_sensor_model_path, time_format=all_args.time_format) ) if new_args.parse: