def __init__(self, port, prefix=None, timestamp=False,
time_format=TIME_FORMAT, filebase=None, eol='\n'):
"""
```
port - UDP port on which to write records.
prefix - If non-empty, prefix to add
timestamp = If True, apply current timestamp to record
time_format - What format to use for timestamp
filebase - Prefix string to be matched (with a following "*") to find
files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud
```
"""
self.port = port
self.prefix = PrefixTransform(prefix) if prefix else None
self.timestamp = TimestampTransform() if timestamp else None
self.time_format = time_format
self.filebase = filebase
# Do we have any files we can actually read from?
if not glob.glob(filebase + '*'):
logging.warning('No files matching "%s*"', filebase)
self.quit_flag = True
return
self.reader = LogfileReader(filebase=filebase, use_timestamps=True,
time_format=self.time_format)
self.slice_n = SliceTransform(fields='1:') # strip off timestamp
self.writer = UDPWriter(port=port, eol=eol)
self.first_time = True
self.quit_flag = False
def run(self):
"""Create the virtual port with socat and start feeding it records from
the designated logfile."""
self.socat_thread = threading.Thread(target=self._run_socat)
self.socat_thread.start()
time.sleep(0.2)
self.reader = LogfileReader(filebase=self.source_file,
use_timestamps=self.use_timestamps)
self.strip = SliceTransform('1:') # strip off the first field)
self.writer = TextFileWriter(self.write_port, truncate=True)
while not self.quit:
record = self.reader.read() # get the next record
logging.debug('SimSerial got: %s', record)
if record is None:
break
record = self.strip.transform(record) # strip the timestamp
if record:
logging.debug('SimSerial writing: %s', record)
self.writer.write(record) # and write it to the virtual port
# If we're here, we got None from our input, and are done. Signal
# for run_socat to exit
self.quit = True
def run(self, loop=False):
# If self.serial_params is None, it means that either read or
# write device already exist, so we shouldn't actually run, or
# we'll destroy them.
if not self.serial_params:
return
"""Create the virtual port with socat and start feeding it records from
the designated logfile. If loop==True, loop when reaching end of input."""
self.socat_thread = threading.Thread(target=self._run_socat,
daemon=True)
self.socat_thread.start()
time.sleep(0.2)
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True,
time_format=self.time_format)
self.slice_n = SliceTransform(
'1:') # strip off the first field (timestamp)
self.writer = TextFileWriter(self.write_port, truncate=True)
logging.info('Starting %s: %s', self.read_port, self.filebase)
while not self.quit:
try:
record = self.reader.read() # get the next record
logging.debug('SimSerial got: %s', record)
# End of input? If loop==True, re-open the logfile from the start
if record is None:
if not loop:
break
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True,
time_format=self.time_format)
record = self.slice_n.transform(record) # strip the timestamp
if not record:
continue
if self.timestamp: # do we want to add a timestamp?
record = self.timestamp.transform(record)
if self.prefix: # do we want to add prefix?
record = self.prefix.transform(record)
logging.debug('SimSerial writing: %s', record)
self.writer.write(record) # and write it to the virtual port
except (OSError, KeyboardInterrupt):
break
# If we're here, we got None from our input, and are done. Signal
# for run_socat to exit
self.quit = True
def test_readline(self):
port = PORT.replace('%DIR%', self.tmpdirname)
sim = SimSerial(port=port, source_file=self.logfile_filename,
use_timestamps=False)
sim_thread = threading.Thread(target=sim.run)
sim_thread.start()
# Give it a moment to get started
time.sleep(0.1)
slice = SliceTransform('1:') # we'll want to strip out timestamp
# Then read from serial port
s = SerialReader(port=port)
for line in SAMPLE_DATA.split('\n'):
data = slice.transform(line)
record = s.read()
logging.debug('data: %s, read: %s', data, record)
self.assertEqual(data, record)
def __init__(self, port, filebase, instrument):
"""
```
port - UDP port on which to write records.
filebase - Prefix string to be matched (with a following "*") to fine
files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud
instrument - Instrument name prefix to add before sendind out on wire
```
"""
self.filebase = filebase
self.reader = LogfileReader(filebase=filebase, use_timestamps=True)
self.slice_n = SliceTransform(
fields='1:') # grab 2nd and subsequent fields
self.timestamp = TimestampTransform()
self.prefix = PrefixTransform(instrument)
self.writer = UDPWriter(port=port)
self.instrument = instrument
self.first_time = True
self.quit_flag = False
def test_use_timestamps(self):
port = PORT.replace('%DIR%', self.tmpdirname)
sim = SimSerial(port=port, source_file=self.logfile_filename)
sim_thread = threading.Thread(target=sim.run)
sim_thread.start()
# Give it a moment to get started
time.sleep(0.1)
slice = SliceTransform('1:') # we'll want to strip out timestamp
# Then read from serial port
s = serial.Serial(port=port)
last_read = time.time() - 0.15
for line in SAMPLE_DATA.split('\n'):
data = slice.transform(line)
record = s.readline().strip().decode('utf-8')
now = time.time()
logging.debug('time diff %f', now - last_read)
self.assertAlmostEqual(now - last_read, 0.26, places=1)
self.assertEqual(data, record)
last_read = now
def run(self, loop=False):
"""Create the virtual port with socat and start feeding it records from
the designated logfile. If loop==True, loop when reaching end of input."""
self.socat_thread = threading.Thread(target=self._run_socat,
daemon=True)
self.socat_thread.start()
time.sleep(0.2)
self.reader = LogfileReader(filebase=self.source_file,
use_timestamps=self.use_timestamps,
time_format=self.time_format)
self.strip = SliceTransform('1:') # strip off the first field)
self.writer = TextFileWriter(self.write_port, truncate=True)
while not self.quit:
try:
record = self.reader.read() # get the next record
logging.debug('SimSerial got: %s', record)
# End of input? If loop==True, re-open the logfile from the start
if record is None:
if not loop:
break
self.reader = LogfileReader(
filebase=self.source_file,
use_timestamps=self.use_timestamps)
record = self.strip.transform(record) # strip the timestamp
if record:
logging.debug('SimSerial writing: %s', record)
self.writer.write(
record) # and write it to the virtual port
except (OSError, KeyboardInterrupt):
break
# If we're here, we got None from our input, and are done. Signal
# for run_socat to exit
self.quit = True
def test_timeout(self):
port = PORT.replace('%DIR%', self.tmpdirname)
sim = SimSerial(port=port, source_file=self.logfile_filename)
sim_thread = threading.Thread(target=sim.run)
sim_thread.start()
# Give it a moment to get started
time.sleep(0.05)
slice = SliceTransform('1:') # we'll want to strip out timestamp
# Then read from serial port
s = SerialReader(port=port, timeout=0.1)
for line in SAMPLE_TIMEOUT:
record = s.read()
logging.debug('data: %s, read: %s', line, record)
self.assertEqual(line, record, msg='Note: this is a time-sensitive '
'test that can fail non-deterministically. If the '
'test fails, try running from the command line, e.g. '
'as "logger/readers/test_serial_reader.py"')
class SimNetwork:
"""Open a network port and feed stored logfile data to it."""
############################
def __init__(self, port, filebase, instrument):
"""
```
port - UDP port on which to write records.
filebase - Prefix string to be matched (with a following "*") to fine
files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud
instrument - Instrument name prefix to add before sendind out on wire
```
"""
self.filebase = filebase
self.reader = LogfileReader(filebase=filebase, use_timestamps=True)
self.slice_n = SliceTransform(
fields='1:') # grab 2nd and subsequent fields
self.timestamp = TimestampTransform()
self.prefix = PrefixTransform(instrument)
self.writer = UDPWriter(port=port)
self.instrument = instrument
self.first_time = True
self.quit_flag = False
############################
def run(self, loop=False):
"""Start reading and writing data. If loop==True, loop when reaching
end of input.
"""
logging.info('Starting %s', self.instrument)
try:
while not self.quit_flag:
record = self.reader.read()
# If we don't have a record, we're (probably) at the end of
# the file. If it's the first time we've tried reading, it
# means we probably didn't get a usable file. Either break out
# (if we're not looping, or if we don't have a usable file),
# or start reading from the beginning (if we are looping and
# have a usable file).
if not record:
if not loop or self.first_time:
break
logging.info('Looping instrument %s', self.instrument)
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True)
continue
# Strip off timestamp and tack on a new one
record = self.slice_n.transform(record)
record = self.timestamp.transform(record)
# Add instrument name back on, and write to specified network
record = self.prefix.transform(record)
self.writer.write(record)
self.first_time = False
except (OSError, KeyboardInterrupt):
self.quit_flag = True
logging.info('Finished %s', self.instrument)
class SimSerial:
"""Create a virtual serial port and feed stored logfile data to it."""
############################
def __init__(self, port, prefix=None, timestamp=False,
time_format=TIME_FORMAT, filebase=None, eol='\n',
baudrate=9600, bytesize=8, parity='N', stopbits=1,
timeout=None, xonxoff=False, rtscts=False, write_timeout=None,
dsrdtr=False, inter_byte_timeout=None, exclusive=None):
"""
Simulate a serial port, feeding it data from the specified file.
```
port - Temporary serial port to create and make available for reading
records.
prefix - If non-empty, prefix name prefix to add
timestamp = If True, apply current timestamp to record
time_format - What format to use for timestamp
```
"""
# We'll create two virtual ports: 'port' and 'port_in'; we will write
# to port_in and read the values back out from port
self.read_port = port
self.write_port = port + '_in'
self.prefix = PrefixTransform(prefix) if prefix else None
self.timestamp = TimestampTransform() if timestamp else None
self.time_format = time_format
self.filebase = filebase
self.serial_params = None
# Complain, but go ahead if read_port or write_port exist.
for path in [self.read_port, self.write_port]:
if os.path.exists(path):
logging.warning('Path %s exists; overwriting!')
# Do we have any files we can actually read from?
if not glob.glob(filebase + '*'):
logging.warning('%s: no files matching "%s*"', prefix, filebase)
return
# Set up our parameters
self.serial_params = {'baudrate': baudrate,
'byteside': bytesize,
'parity': parity,
'stopbits': stopbits,
'timeout': timeout,
'xonxoff': xonxoff,
'rtscts': rtscts,
'write_timeout': write_timeout,
'dsrdtr': dsrdtr,
'inter_byte_timeout': inter_byte_timeout,
'exclusive': exclusive}
self.quit = False
# Finally, find path to socat executable
self.socat_path = None
for socat_path in ['/usr/bin/socat', '/usr/local/bin/socat']:
if os.path.exists(socat_path) and os.path.isfile(socat_path):
self.socat_path = socat_path
if not self.socat_path:
raise NameError('Executable "socat" not found on path. Please refer '
'to installation guide to install socat.')
############################
def _run_socat(self):
"""Internal: run the actual command."""
verbose = '-d'
write_port_params = 'pty,link=%s,raw,echo=0' % self.write_port
read_port_params = 'pty,link=%s,raw,echo=0' % self.read_port
cmd = [self.socat_path,
verbose,
#verbose, # repeating makes it more verbose
read_port_params,
write_port_params,
]
try:
# Run socat process using Popen, checking every second or so whether
# it's died (poll() != None) or we've gotten a quit signal.
logging.info('Calling: %s', ' '.join(cmd))
socat_process = subprocess.Popen(cmd)
while not self.quit and not socat_process.poll():
try:
socat_process.wait(1)
except subprocess.TimeoutExpired:
pass
except Exception as e:
logging.error('ERROR: socat command: %s', e)
# If here, process has terminated, or we've seen self.quit. We
# want both to be true: if we've terminated, set self.quit so that
# 'run' loop can exit. If self.quit, terminate process.
if self.quit:
socat_process.kill()
else:
self.quit = True
logging.info('Finished: %s', ' '.join(cmd))
############################
def run(self, loop=False):
# If self.serial_params is None, it means that either read or
# write device already exist, so we shouldn't actually run, or
# we'll destroy them.
if not self.serial_params:
return
"""Create the virtual port with socat and start feeding it records from
the designated logfile. If loop==True, loop when reaching end of input."""
self.socat_thread = threading.Thread(target=self._run_socat, daemon=True)
self.socat_thread.start()
time.sleep(0.2)
self.reader = LogfileReader(filebase=self.filebase, use_timestamps=True,
time_format=self.time_format)
self.slice_n = SliceTransform('1:') # strip off the first field (timestamp)
self.writer = TextFileWriter(self.write_port, truncate=True)
logging.info('Starting %s: %s', self.read_port, self.filebase)
while not self.quit:
try:
record = self.reader.read() # get the next record
logging.debug('SimSerial got: %s', record)
# End of input? If loop==True, re-open the logfile from the start
if record is None:
if not loop:
break
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True,
time_format=self.time_format)
record = self.slice_n.transform(record) # strip the timestamp
if not record:
continue
if self.timestamp: # do we want to add a timestamp?
record = self.timestamp.transform(record)
if self.prefix: # do we want to add prefix?
record = self.prefix.transform(record)
logging.debug('SimSerial writing: %s', record)
self.writer.write(record) # and write it to the virtual port
except (OSError, KeyboardInterrupt):
break
# If we're here, we got None from our input, and are done. Signal
# for run_socat to exit
self.quit = True
class SimUDP:
"""Open a network port and feed stored logfile data to it."""
############################
def __init__(self, port, prefix=None, timestamp=False,
time_format=TIME_FORMAT, filebase=None, eol='\n'):
"""
```
port - UDP port on which to write records.
prefix - If non-empty, prefix to add
timestamp = If True, apply current timestamp to record
time_format - What format to use for timestamp
filebase - Prefix string to be matched (with a following "*") to find
files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud
```
"""
self.port = port
self.prefix = PrefixTransform(prefix) if prefix else None
self.timestamp = TimestampTransform() if timestamp else None
self.time_format = time_format
self.filebase = filebase
# Do we have any files we can actually read from?
if not glob.glob(filebase + '*'):
logging.warning('No files matching "%s*"', filebase)
self.quit_flag = True
return
self.reader = LogfileReader(filebase=filebase, use_timestamps=True,
time_format=self.time_format)
self.slice_n = SliceTransform(fields='1:') # strip off timestamp
self.writer = UDPWriter(port=port, eol=eol)
self.first_time = True
self.quit_flag = False
############################
def run(self, loop=False):
"""Start reading and writing data. If loop==True, loop when reaching
end of input.
"""
logging.info('Starting %s: %s', self.port, self.filebase)
try:
while not self.quit_flag:
record = self.reader.read()
# If we don't have a record, we're (probably) at the end of
# the file. If it's the first time we've tried reading, it
# means we probably didn't get a usable file. Either break out
# (if we're not looping, or if we don't have a usable file),
# or start reading from the beginning (if we are looping and
# have a usable file).
if not record:
if not loop or self.first_time:
break
logging.info('Looping instrument %s', self.prefix)
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True)
continue
# Strip off timestamp
record = self.slice_n.transform(record)
if not record:
continue
record = record.strip()
if not record:
continue
if self.timestamp: # do we want to add a timestamp?
record = self.timestamp.transform(record)
if self.prefix: # do we want to add prefix?
record = self.prefix.transform(record)
self.writer.write(record)
self.first_time = False
except (OSError, KeyboardInterrupt):
self.quit_flag = True
logging.info('Finished %s', self.prefix)
def test_default(self):
with self.assertLogs(logging.getLogger(), logging.ERROR):
with self.assertRaises(ValueError):
transform = SliceTransform('1.5')
with self.assertLogs(logging.getLogger(), logging.ERROR):
with self.assertRaises(ValueError):
transform = SliceTransform('3,4,a')
alpha = 'a b c d e f g h i j k l'
transform = SliceTransform('1')
self.assertEqual(transform.transform(alpha), 'b')
transform = SliceTransform('3:5')
self.assertEqual(transform.transform(alpha), 'd e')
transform = SliceTransform('-1')
self.assertEqual(transform.transform(alpha), 'l')
transform = SliceTransform('-7, -3:-2')
self.assertEqual(transform.transform(alpha), 'f j')
transform = SliceTransform(':3,5:7,9,11:')
self.assertEqual(transform.transform(alpha), 'a b c f g j l')
transform = SliceTransform(':')
self.assertEqual(transform.transform(alpha), alpha)
transform = SliceTransform('')
self.assertEqual(transform.transform(alpha), alpha)
user=user, password=password))
# SerialReader is a little more complicated than other readers
# because it can take so many parameters. Use the kwargs trick to
# pass them all in.
if new_args.serial:
kwargs = {}
for pair in new_args.serial.split(','):
(key, value) = pair.split('=')
kwargs[key] = value
readers.append(SerialReader(**kwargs))
##########################
# Transforms
if new_args.slice:
transforms.append(SliceTransform(new_args.slice,
all_args.slice_separator))
if new_args.timestamp:
transforms.append(TimestampTransform(time_format=all_args.time_format))
if new_args.prefix:
transforms.append(PrefixTransform(new_args.prefix))
if new_args.extract:
transforms.append(ExtractFieldTransform(new_args.extract))
if new_args.regex_filter:
transforms.append(RegexFilterTransform(new_args.regex_filter))
if new_args.qc_filter:
transforms.append(QCFilterTransform(new_args.qc_filter))
if new_args.parse_nmea:
transforms.append(
ParseNMEATransform(
message_path=all_args.parse_nmea_message_path,
sensor_path=all_args.parse_nmea_sensor_path,
# SerialReader is a little more complicated than other readers
# because it can take so many parameters. Use the kwargs trick to
# pass them all in.
if new_args.serial:
kwargs = {}
for pair in new_args.serial.split(','):
(key, value) = pair.split('=')
kwargs[key] = value
readers.append(SerialReader(**kwargs))
##########################
# Transforms
if new_args.slice:
transforms.append(
SliceTransform(new_args.slice, all_args.slice_separator))
if new_args.timestamp:
transforms.append(
TimestampTransform(time_format=all_args.time_format))
if new_args.prefix:
transforms.append(PrefixTransform(new_args.prefix))
if new_args.regex_filter:
transforms.append(RegexFilterTransform(new_args.regex_filter))
if new_args.qc_filter:
transforms.append(QCFilterTransform(new_args.qc_filter))
if new_args.parse_nmea:
transforms.append(
ParseNMEATransform(time_format=all_args.time_format))
if new_args.aggregate_xml:
transforms.append(
XMLAggregatorTransform(new_args.aggregate_xml))
class SimSerial:
"""Create a virtual serial port and feed stored logfile data to it."""
############################
def __init__(self,
port,
source_file,
use_timestamps=True,
baudrate=9600,
bytesize=8,
parity='N',
stopbits=1,
timeout=None,
xonxoff=False,
rtscts=False,
write_timeout=None,
dsrdtr=False,
inter_byte_timeout=None,
exclusive=None):
"""Takes source file, whether to deliver data at rate indicated by
timestamps, and the standard parameters that a serial port takes."""
self.source_file = source_file
self.use_timestamps = use_timestamps
# We'll create two virtual ports: 'port' and 'port_in'; we will write
# to port_in and read the values back out from port
self.read_port = port
self.write_port = port + '_in'
self.serial_params = {
'baudrate': baudrate,
'byteside': bytesize,
'parity': parity,
'stopbits': stopbits,
'timeout': timeout,
'xonxoff': xonxoff,
'rtscts': rtscts,
'write_timeout': write_timeout,
'dsrdtr': dsrdtr,
'inter_byte_timeout': inter_byte_timeout,
'exclusive': exclusive
}
self.quit = False
# Finally, check that our needed 'socat' actually exists
if not subprocess.run(['which', 'socat'],
stdout=subprocess.PIPE).stdout:
raise NameError(
'Executable "socat" not found on path. Please refer '
'to installation guide to install socat.')
############################
def _run_socat(self):
"""Internal: run the actual command."""
verbose = '-d'
write_port_params = 'pty,link=%s,raw,echo=0' % self.write_port
read_port_params = 'pty,link=%s,raw,echo=0' % self.read_port
cmd = [
'/usr/bin/env',
'socat',
verbose,
#verbose, # repeating makes it more verbose
read_port_params,
write_port_params,
]
try:
# Run socat process using Popen, checking every second or so whether
# it's died (poll() != None) or we've gotten a quit signal.
logging.info('Calling: %s', ' '.join(cmd))
socat_process = subprocess.Popen(cmd)
while not self.quit and not socat_process.poll():
try:
socat_process.wait(1)
except subprocess.TimeoutExpired:
pass
except Exception as e:
logging.error('ERROR: socat command: %s', e)
# If here, process has terminated, or we've seen self.quit. We
# want both to be true: if we've terminated, set self.quit so that
# 'run' loop can exit. If self.quit, terminate process.
if self.quit:
socat_process.kill()
else:
self.quit = True
logging.info('Finished: %s', ' '.join(cmd))
############################
def run(self):
"""Create the virtual port with socat and start feeding it records from
the designated logfile."""
self.socat_thread = threading.Thread(target=self._run_socat)
self.socat_thread.start()
time.sleep(0.2)
self.reader = LogfileReader(filebase=self.source_file,
use_timestamps=self.use_timestamps)
self.strip = SliceTransform('1:') # strip off the first field)
self.writer = TextFileWriter(self.write_port, truncate=True)
while not self.quit:
record = self.reader.read() # get the next record
logging.debug('SimSerial got: %s', record)
if record is None:
break
record = self.strip.transform(record) # strip the timestamp
if record:
logging.debug('SimSerial writing: %s', record)
self.writer.write(record) # and write it to the virtual port
# If we're here, we got None from our input, and are done. Signal
# for run_socat to exit
self.quit = True
