def test_prefix(self):
prefix_1 = PrefixTransform('p1')
prefix_2 = PrefixTransform('p2')
f1_name = self.tmpdirname + '/f1'
f2_name = self.tmpdirname + '/f2'
writer = ComposedWriter(
transforms=[prefix_1, prefix_2],
writers=[TextFileWriter(f1_name),
TextFileWriter(f2_name)])
f1 = open(f1_name, 'r')
f2 = open(f2_name, 'r')
for line in SAMPLE_DATA:
writer.write(line)
time.sleep(0.1)
f1_line = f1.readline().rstrip()
f2_line = f2.readline().rstrip()
logging.info('wrote: "%s", f1: "%s", f2: "%s"', line, f1_line,
f2_line)
self.assertEqual('p2 p1 ' + line, f1_line)
self.assertEqual('p2 p1 ' + line, f2_line)
def test_read_all_write_one(self):
readers = []
for tmpfilename in self.tmpfilenames:
readers.append(TextFileReader(tmpfilename, interval=0.2))
transforms = [PrefixTransform('prefix_1'), PrefixTransform('prefix_2')]
outfilename = self.tmpdirname + '/f_out'
writers = [TextFileWriter(outfilename)]
listener = Listener(readers, transforms, writers)
listener.run()
out_lines = []
with open(outfilename, 'r') as f:
for line in f.readlines():
out_lines.append(line.rstrip())
out_lines.sort()
source_lines = []
for f in SAMPLE_DATA:
source_lines.extend(
['prefix_2 prefix_1 ' + f for f in SAMPLE_DATA[f]])
source_lines.sort()
logging.debug('out: %s, source: %s', out_lines, source_lines)
self.assertEqual(out_lines, source_lines)
def __init__(self, port, prefix=None, timestamp=False,
time_format=TIME_FORMAT, filebase=None, eol='\n'):
"""
```
port - UDP port on which to write records.
prefix - If non-empty, prefix to add
timestamp = If True, apply current timestamp to record
time_format - What format to use for timestamp
filebase - Prefix string to be matched (with a following "*") to find
files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud
```
"""
self.port = port
self.prefix = PrefixTransform(prefix) if prefix else None
self.timestamp = TimestampTransform() if timestamp else None
self.time_format = time_format
self.filebase = filebase
# Do we have any files we can actually read from?
if not glob.glob(filebase + '*'):
logging.warning('No files matching "%s*"', filebase)
self.quit_flag = True
return
self.reader = LogfileReader(filebase=filebase, use_timestamps=True,
time_format=self.time_format)
self.slice_n = SliceTransform(fields='1:') # strip off timestamp
self.writer = UDPWriter(port=port, eol=eol)
self.first_time = True
self.quit_flag = False
def test_all_files(self):
# Use TextFileReader's 'interval' flag to make sure we interleave
# reads the way we expect. Also make sure transforms get applied
# in proper order.
readers = []
for tmpfilename in self.tmpfilenames:
readers.append(TextFileReader(tmpfilename, interval=0.2))
#readers.append(TextFileReader()) # read from stdin
prefix_1 = PrefixTransform('prefix_1')
prefix_2 = PrefixTransform('prefix_2')
reader = ComposedReader(readers, [prefix_1, prefix_2])
# Clunkly quick way of slicing lines
i = 0
expected_lines = []
while True:
next_lines = []
for f in sorted(SAMPLE_DATA):
if i < len(SAMPLE_DATA[f]):
line = 'prefix_2 prefix_1 ' + SAMPLE_DATA[f][i]
next_lines.append(line)
if next_lines:
expected_lines.append(next_lines)
i += 1
else:
break
logging.debug('Expected lines %s', expected_lines)
# Next line from each of the files can come in arbitrary order,
# but within the file, lines should arrive in order, and we
# should receive first line from each file before we receive
# next line from any of them.
while expected_lines:
next_lines = expected_lines.pop(0)
while next_lines:
record = reader.read()
logging.info('read: %s; expected one of: %s', record,
next_lines)
self.assertTrue(record in next_lines)
if record in next_lines:
next_lines.remove(record)
self.assertEqual(None, reader.read())
def test_default(self):
transform = PrefixTransform('prefix')
self.assertIsNone(transform.transform(None))
self.assertEqual(transform.transform('foo'), 'prefix foo')
transform = PrefixTransform('prefix', sep='\t')
self.assertEqual(transform.transform('foo'), 'prefix\tfoo')
def __init__(self, port, filebase, instrument):
"""
```
port - UDP port on which to write records.
filebase - Prefix string to be matched (with a following "*") to fine
files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud
instrument - Instrument name prefix to add before sendind out on wire
```
"""
self.filebase = filebase
self.reader = LogfileReader(filebase=filebase, use_timestamps=True)
self.slice_n = SliceTransform(
fields='1:') # grab 2nd and subsequent fields
self.timestamp = TimestampTransform()
self.prefix = PrefixTransform(instrument)
self.writer = UDPWriter(port=port)
self.instrument = instrument
self.first_time = True
self.quit_flag = False
def test_check_format(self):
f1_name = self.tmpdirname + '/f1'
f2_name = self.tmpdirname + '/f2'
# This should be okay
ComposedWriter(
transforms=[PrefixTransform('prefix')],
writers=[TextFileWriter(f1_name),
TextFileWriter(f2_name)],
check_format=True)
# Should raise an error if formats are not compatible
with self.assertLogs(logging.getLogger(), logging.ERROR):
with self.assertRaises(ValueError):
ComposedWriter(
transforms=[ParseNMEATransform()],
writers=[TextFileWriter(f1_name),
TextFileWriter(f2_name)],
check_format=True)
class SimNetwork:
"""Open a network port and feed stored logfile data to it."""
############################
def __init__(self, port, filebase, instrument):
"""
```
port - UDP port on which to write records.
filebase - Prefix string to be matched (with a following "*") to fine
files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud
instrument - Instrument name prefix to add before sendind out on wire
```
"""
self.filebase = filebase
self.reader = LogfileReader(filebase=filebase, use_timestamps=True)
self.slice_n = SliceTransform(
fields='1:') # grab 2nd and subsequent fields
self.timestamp = TimestampTransform()
self.prefix = PrefixTransform(instrument)
self.writer = UDPWriter(port=port)
self.instrument = instrument
self.first_time = True
self.quit_flag = False
############################
def run(self, loop=False):
"""Start reading and writing data. If loop==True, loop when reaching
end of input.
"""
logging.info('Starting %s', self.instrument)
try:
while not self.quit_flag:
record = self.reader.read()
# If we don't have a record, we're (probably) at the end of
# the file. If it's the first time we've tried reading, it
# means we probably didn't get a usable file. Either break out
# (if we're not looping, or if we don't have a usable file),
# or start reading from the beginning (if we are looping and
# have a usable file).
if not record:
if not loop or self.first_time:
break
logging.info('Looping instrument %s', self.instrument)
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True)
continue
# Strip off timestamp and tack on a new one
record = self.slice_n.transform(record)
record = self.timestamp.transform(record)
# Add instrument name back on, and write to specified network
record = self.prefix.transform(record)
self.writer.write(record)
self.first_time = False
except (OSError, KeyboardInterrupt):
self.quit_flag = True
logging.info('Finished %s', self.instrument)
def __init__(self, port, prefix=None, timestamp=False,
time_format=TIME_FORMAT, filebase=None, eol='\n',
baudrate=9600, bytesize=8, parity='N', stopbits=1,
timeout=None, xonxoff=False, rtscts=False, write_timeout=None,
dsrdtr=False, inter_byte_timeout=None, exclusive=None):
"""
Simulate a serial port, feeding it data from the specified file.
```
port - Temporary serial port to create and make available for reading
records.
prefix - If non-empty, prefix name prefix to add
timestamp = If True, apply current timestamp to record
time_format - What format to use for timestamp
```
"""
# We'll create two virtual ports: 'port' and 'port_in'; we will write
# to port_in and read the values back out from port
self.read_port = port
self.write_port = port + '_in'
self.prefix = PrefixTransform(prefix) if prefix else None
self.timestamp = TimestampTransform() if timestamp else None
self.time_format = time_format
self.filebase = filebase
self.serial_params = None
# Complain, but go ahead if read_port or write_port exist.
for path in [self.read_port, self.write_port]:
if os.path.exists(path):
logging.warning('Path %s exists; overwriting!')
# Do we have any files we can actually read from?
if not glob.glob(filebase + '*'):
logging.warning('%s: no files matching "%s*"', prefix, filebase)
return
# Set up our parameters
self.serial_params = {'baudrate': baudrate,
'byteside': bytesize,
'parity': parity,
'stopbits': stopbits,
'timeout': timeout,
'xonxoff': xonxoff,
'rtscts': rtscts,
'write_timeout': write_timeout,
'dsrdtr': dsrdtr,
'inter_byte_timeout': inter_byte_timeout,
'exclusive': exclusive}
self.quit = False
# Finally, find path to socat executable
self.socat_path = None
for socat_path in ['/usr/bin/socat', '/usr/local/bin/socat']:
if os.path.exists(socat_path) and os.path.isfile(socat_path):
self.socat_path = socat_path
if not self.socat_path:
raise NameError('Executable "socat" not found on path. Please refer '
'to installation guide to install socat.')
class SimSerial:
"""Create a virtual serial port and feed stored logfile data to it."""
############################
def __init__(self, port, prefix=None, timestamp=False,
time_format=TIME_FORMAT, filebase=None, eol='\n',
baudrate=9600, bytesize=8, parity='N', stopbits=1,
timeout=None, xonxoff=False, rtscts=False, write_timeout=None,
dsrdtr=False, inter_byte_timeout=None, exclusive=None):
"""
Simulate a serial port, feeding it data from the specified file.
```
port - Temporary serial port to create and make available for reading
records.
prefix - If non-empty, prefix name prefix to add
timestamp = If True, apply current timestamp to record
time_format - What format to use for timestamp
```
"""
# We'll create two virtual ports: 'port' and 'port_in'; we will write
# to port_in and read the values back out from port
self.read_port = port
self.write_port = port + '_in'
self.prefix = PrefixTransform(prefix) if prefix else None
self.timestamp = TimestampTransform() if timestamp else None
self.time_format = time_format
self.filebase = filebase
self.serial_params = None
# Complain, but go ahead if read_port or write_port exist.
for path in [self.read_port, self.write_port]:
if os.path.exists(path):
logging.warning('Path %s exists; overwriting!')
# Do we have any files we can actually read from?
if not glob.glob(filebase + '*'):
logging.warning('%s: no files matching "%s*"', prefix, filebase)
return
# Set up our parameters
self.serial_params = {'baudrate': baudrate,
'byteside': bytesize,
'parity': parity,
'stopbits': stopbits,
'timeout': timeout,
'xonxoff': xonxoff,
'rtscts': rtscts,
'write_timeout': write_timeout,
'dsrdtr': dsrdtr,
'inter_byte_timeout': inter_byte_timeout,
'exclusive': exclusive}
self.quit = False
# Finally, find path to socat executable
self.socat_path = None
for socat_path in ['/usr/bin/socat', '/usr/local/bin/socat']:
if os.path.exists(socat_path) and os.path.isfile(socat_path):
self.socat_path = socat_path
if not self.socat_path:
raise NameError('Executable "socat" not found on path. Please refer '
'to installation guide to install socat.')
############################
def _run_socat(self):
"""Internal: run the actual command."""
verbose = '-d'
write_port_params = 'pty,link=%s,raw,echo=0' % self.write_port
read_port_params = 'pty,link=%s,raw,echo=0' % self.read_port
cmd = [self.socat_path,
verbose,
#verbose, # repeating makes it more verbose
read_port_params,
write_port_params,
]
try:
# Run socat process using Popen, checking every second or so whether
# it's died (poll() != None) or we've gotten a quit signal.
logging.info('Calling: %s', ' '.join(cmd))
socat_process = subprocess.Popen(cmd)
while not self.quit and not socat_process.poll():
try:
socat_process.wait(1)
except subprocess.TimeoutExpired:
pass
except Exception as e:
logging.error('ERROR: socat command: %s', e)
# If here, process has terminated, or we've seen self.quit. We
# want both to be true: if we've terminated, set self.quit so that
# 'run' loop can exit. If self.quit, terminate process.
if self.quit:
socat_process.kill()
else:
self.quit = True
logging.info('Finished: %s', ' '.join(cmd))
############################
def run(self, loop=False):
# If self.serial_params is None, it means that either read or
# write device already exist, so we shouldn't actually run, or
# we'll destroy them.
if not self.serial_params:
return
"""Create the virtual port with socat and start feeding it records from
the designated logfile. If loop==True, loop when reaching end of input."""
self.socat_thread = threading.Thread(target=self._run_socat, daemon=True)
self.socat_thread.start()
time.sleep(0.2)
self.reader = LogfileReader(filebase=self.filebase, use_timestamps=True,
time_format=self.time_format)
self.slice_n = SliceTransform('1:') # strip off the first field (timestamp)
self.writer = TextFileWriter(self.write_port, truncate=True)
logging.info('Starting %s: %s', self.read_port, self.filebase)
while not self.quit:
try:
record = self.reader.read() # get the next record
logging.debug('SimSerial got: %s', record)
# End of input? If loop==True, re-open the logfile from the start
if record is None:
if not loop:
break
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True,
time_format=self.time_format)
record = self.slice_n.transform(record) # strip the timestamp
if not record:
continue
if self.timestamp: # do we want to add a timestamp?
record = self.timestamp.transform(record)
if self.prefix: # do we want to add prefix?
record = self.prefix.transform(record)
logging.debug('SimSerial writing: %s', record)
self.writer.write(record) # and write it to the virtual port
except (OSError, KeyboardInterrupt):
break
# If we're here, we got None from our input, and are done. Signal
# for run_socat to exit
self.quit = True
class SimUDP:
"""Open a network port and feed stored logfile data to it."""
############################
def __init__(self, port, prefix=None, timestamp=False,
time_format=TIME_FORMAT, filebase=None, eol='\n'):
"""
```
port - UDP port on which to write records.
prefix - If non-empty, prefix to add
timestamp = If True, apply current timestamp to record
time_format - What format to use for timestamp
filebase - Prefix string to be matched (with a following "*") to find
files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud
```
"""
self.port = port
self.prefix = PrefixTransform(prefix) if prefix else None
self.timestamp = TimestampTransform() if timestamp else None
self.time_format = time_format
self.filebase = filebase
# Do we have any files we can actually read from?
if not glob.glob(filebase + '*'):
logging.warning('No files matching "%s*"', filebase)
self.quit_flag = True
return
self.reader = LogfileReader(filebase=filebase, use_timestamps=True,
time_format=self.time_format)
self.slice_n = SliceTransform(fields='1:') # strip off timestamp
self.writer = UDPWriter(port=port, eol=eol)
self.first_time = True
self.quit_flag = False
############################
def run(self, loop=False):
"""Start reading and writing data. If loop==True, loop when reaching
end of input.
"""
logging.info('Starting %s: %s', self.port, self.filebase)
try:
while not self.quit_flag:
record = self.reader.read()
# If we don't have a record, we're (probably) at the end of
# the file. If it's the first time we've tried reading, it
# means we probably didn't get a usable file. Either break out
# (if we're not looping, or if we don't have a usable file),
# or start reading from the beginning (if we are looping and
# have a usable file).
if not record:
if not loop or self.first_time:
break
logging.info('Looping instrument %s', self.prefix)
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True)
continue
# Strip off timestamp
record = self.slice_n.transform(record)
if not record:
continue
record = record.strip()
if not record:
continue
if self.timestamp: # do we want to add a timestamp?
record = self.timestamp.transform(record)
if self.prefix: # do we want to add prefix?
record = self.prefix.transform(record)
self.writer.write(record)
self.first_time = False
except (OSError, KeyboardInterrupt):
self.quit_flag = True
logging.info('Finished %s', self.prefix)
action='count',
help='Increase output verbosity')
args = parser.parse_args()
LOGGING_FORMAT = '%(asctime)-15s %(message)s'
logging.basicConfig(format=LOGGING_FORMAT)
LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG}
args.verbosity = min(args.verbosity, max(LOG_LEVELS))
logging.getLogger().setLevel(LOG_LEVELS[args.verbosity])
reader = TextFileReader(file_spec=args.read, tail=args.tail)
writer = TextFileWriter(filename=args.write)
if args.prefix:
prefix_transform = PrefixTransform(args.prefix)
if args.timestamp:
timestamp_transform = TimestampTransform()
while True:
record = reader.read()
now = time.time()
logging.info('Got record: %s', record)
if record is None:
if not args.tail:
break
else:
if args.timestamp:
if new_args.serial:
kwargs = {}
for pair in new_args.serial.split(','):
(key, value) = pair.split('=')
kwargs[key] = value
readers.append(SerialReader(**kwargs))
##########################
# Transforms
if new_args.slice:
transforms.append(SliceTransform(new_args.slice,
all_args.slice_separator))
if new_args.timestamp:
transforms.append(TimestampTransform(time_format=all_args.time_format))
if new_args.prefix:
transforms.append(PrefixTransform(new_args.prefix))
if new_args.extract:
transforms.append(ExtractFieldTransform(new_args.extract))
if new_args.regex_filter:
transforms.append(RegexFilterTransform(new_args.regex_filter))
if new_args.qc_filter:
transforms.append(QCFilterTransform(new_args.qc_filter))
if new_args.parse_nmea:
transforms.append(
ParseNMEATransform(
message_path=all_args.parse_nmea_message_path,
sensor_path=all_args.parse_nmea_sensor_path,
sensor_model_path=all_args.parse_nmea_sensor_model_path,
time_format=all_args.time_format)
)
if new_args.parse:
