def run(self, loop=False):
# If self.serial_params is None, it means that either read or
# write device already exist, so we shouldn't actually run, or
# we'll destroy them.
if not self.serial_params:
return
"""Create the virtual port with socat and start feeding it records from
the designated logfile. If loop==True, loop when reaching end of input."""
self.socat_thread = threading.Thread(target=self._run_socat,
daemon=True)
self.socat_thread.start()
time.sleep(0.2)
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True,
record_format=self.record_format,
time_format=self.time_format)
self.writer = TextFileWriter(self.write_port, truncate=True)
logging.info('Starting %s: %s', self.read_port, self.filebase)
while not self.quit:
try:
record = self.reader.read() # get the next record
logging.debug('SimSerial got: %s', record)
# End of input? If loop==True, re-open the logfile from the start
if record is None:
if not loop:
break
self.reader = LogfileReader(
filebase=self.filebase,
use_timestamps=True,
record_format=self.record_format,
time_format=self.time_format)
# We've now got a record. Try parsing timestamp off it
try:
parsed_record = self.compiled_record_format.parse(
record).named
record = parsed_record['record']
# We had a problem parsing. Discard record and try reading next one.
except (KeyError, ValueError, AttributeError):
logging.warning('Unable to parse record into "%s"',
self.record_format)
logging.warning('Record: %s', record)
continue
if not record:
continue
logging.debug('SimSerial writing: %s', record)
self.writer.write(record) # and write it to the virtual port
except (OSError, KeyboardInterrupt):
break
# If we're here, we got None from our input, and are done. Signal
# for run_socat to exit
self.quit = True
def run(self):
"""Create the virtual port with socat and start feeding it records from
the designated logfile."""
self.socat_thread = threading.Thread(target=self._run_socat)
self.socat_thread.start()
time.sleep(0.2)
self.reader = LogfileReader(filebase=self.source_file,
use_timestamps=self.use_timestamps)
self.strip = SliceTransform('1:') # strip off the first field)
self.writer = TextFileWriter(self.write_port, truncate=True)
while not self.quit:
record = self.reader.read() # get the next record
logging.debug('SimSerial got: %s', record)
if record is None:
break
record = self.strip.transform(record) # strip the timestamp
if record:
logging.debug('SimSerial writing: %s', record)
self.writer.write(record) # and write it to the virtual port
# If we're here, we got None from our input, and are done. Signal
# for run_socat to exit
self.quit = True
def __init__(self, port, prefix=None, timestamp=False,
time_format=TIME_FORMAT, filebase=None, eol='\n'):
"""
```
port - UDP port on which to write records.
prefix - If non-empty, prefix to add
timestamp = If True, apply current timestamp to record
time_format - What format to use for timestamp
filebase - Prefix string to be matched (with a following "*") to find
files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud
```
"""
self.port = port
self.prefix = PrefixTransform(prefix) if prefix else None
self.timestamp = TimestampTransform() if timestamp else None
self.time_format = time_format
self.filebase = filebase
# Do we have any files we can actually read from?
if not glob.glob(filebase + '*'):
logging.warning('No files matching "%s*"', filebase)
self.quit_flag = True
return
self.reader = LogfileReader(filebase=filebase, use_timestamps=True,
time_format=self.time_format)
self.slice_n = SliceTransform(fields='1:') # strip off timestamp
self.writer = UDPWriter(port=port, eol=eol)
self.first_time = True
self.quit_flag = False
def test_use_timestamp(self):
with tempfile.TemporaryDirectory() as tmpdirname:
logging.info('created temporary directory "%s"', tmpdirname)
tmpfilename = tmpdirname + '/mylog-2017-02-02'
sample_lines = SAMPLE_DATA.split('\n')
create_file(tmpfilename, sample_lines)
reader = LogfileReader(tmpfilename, use_timestamps=True)
for line in sample_lines:
self.assertEqual(line, reader.read())
self.assertEqual(None, reader.read())
def run(self, loop=False):
# If self.serial_params is None, it means that either read or
# write device already exist, so we shouldn't actually run, or
# we'll destroy them.
if not self.serial_params:
return
"""Create the virtual port with socat and start feeding it records from
the designated logfile. If loop==True, loop when reaching end of input."""
self.socat_thread = threading.Thread(target=self._run_socat,
daemon=True)
self.socat_thread.start()
time.sleep(0.2)
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True,
time_format=self.time_format)
self.slice_n = SliceTransform(
'1:') # strip off the first field (timestamp)
self.writer = TextFileWriter(self.write_port, truncate=True)
logging.info('Starting %s: %s', self.read_port, self.filebase)
while not self.quit:
try:
record = self.reader.read() # get the next record
logging.debug('SimSerial got: %s', record)
# End of input? If loop==True, re-open the logfile from the start
if record is None:
if not loop:
break
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True,
time_format=self.time_format)
record = self.slice_n.transform(record) # strip the timestamp
if not record:
continue
if self.timestamp: # do we want to add a timestamp?
record = self.timestamp.transform(record)
if self.prefix: # do we want to add prefix?
record = self.prefix.transform(record)
logging.debug('SimSerial writing: %s', record)
self.writer.write(record) # and write it to the virtual port
except (OSError, KeyboardInterrupt):
break
# If we're here, we got None from our input, and are done. Signal
# for run_socat to exit
self.quit = True
def run(self, loop=False):
"""Start reading and writing data. If loop==True, loop when reaching
end of input.
"""
logging.info('Starting %s: %s', self.port, self.filebase)
try:
while not self.quit_flag:
record = self.reader.read()
# If we don't have a record, we're (probably) at the end of
# the file. If it's the first time we've tried reading, it
# means we probably didn't get a usable file. Either break out
# (if we're not looping, or if we don't have a usable file),
# or start reading from the beginning (if we are looping and
# have a usable file).
if not record:
if not loop or self.first_time:
break
logging.info('Looping instrument %s', self.prefix)
self.reader = LogfileReader(
filebase=self.filebase,
record_format=self.record_format,
use_timestamps=True)
continue
# We've now got a record. Try parsing timestamp off it
try:
parsed_record = self.compiled_record_format.parse(
record).named
record = parsed_record['record']
# We had a problem parsing. Discard record and try reading next one.
except (KeyError, ValueError, AttributeError):
logging.warning('Unable to parse record into "%s"',
self.record_format)
logging.warning('Record: %s', record)
continue
if not record:
continue
self.writer.write(record)
self.first_time = False
except (OSError, KeyboardInterrupt):
self.quit_flag = True
logging.info('Finished %s', self.prefix)
def test_tail_false(self):
# Don't specify 'tail' and expect there to be no data
with tempfile.TemporaryDirectory() as tmpdirname:
logging.info('created temporary directory "%s"', tmpdirname)
# Create a file slowly, one line at a time
target = 'mylogfile'
tmpfilename = tmpdirname + '/' + target
sample_lines = SAMPLE_DATA.split('\n')
threading.Thread(target=create_file,
args=(tmpfilename, sample_lines, 0.25)).start()
time.sleep(0.05) # let the thread get started
# Read, and wait for lines to come
reader = LogfileReader(tmpfilename, tail=False)
self.assertEqual(None, reader.read())
def test_interval(self):
with tempfile.TemporaryDirectory() as tmpdirname:
logging.info('created temporary directory "%s"', tmpdirname)
tmpfilename = tmpdirname + '/mylog-2017-02-02'
sample_lines = SAMPLE_DATA.split('\n')
create_file(tmpfilename, sample_lines)
interval = 0.2
reader = LogfileReader(tmpfilename, interval=interval)
then = 0
for line in sample_lines:
self.assertEqual(line, reader.read())
now = time.time()
if then:
self.assertAlmostEqual(now - then, interval, places=1)
then = now
self.assertEqual(None, reader.read())
def run(self, loop=False):
"""Start reading and writing data. If loop==True, loop when reaching
end of input.
"""
logging.info('Starting %s: %s', self.port, self.filebase)
try:
while not self.quit_flag:
record = self.reader.read()
# If we don't have a record, we're (probably) at the end of
# the file. If it's the first time we've tried reading, it
# means we probably didn't get a usable file. Either break out
# (if we're not looping, or if we don't have a usable file),
# or start reading from the beginning (if we are looping and
# have a usable file).
if not record:
if not loop or self.first_time:
break
logging.info('Looping instrument %s', self.prefix)
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True)
continue
# Strip off timestamp
record = self.slice_n.transform(record)
if not record:
continue
record = record.strip()
if not record:
continue
if self.timestamp: # do we want to add a timestamp?
record = self.timestamp.transform(record)
if self.prefix: # do we want to add prefix?
record = self.prefix.transform(record)
self.writer.write(record)
self.first_time = False
except (OSError, KeyboardInterrupt):
self.quit_flag = True
logging.info('Finished %s', self.prefix)
def test_tail_true(self):
# Do the same thing as test_tail_false, but specify tail=True. We should
# now get all the lines that are eventually written to the file.
with tempfile.TemporaryDirectory() as tmpdirname:
logging.info('created temporary directory "%s"', tmpdirname)
# Create a file slowly, one line at a time
target = 'mylogfile'
tmpfilename = tmpdirname + '/' + target
sample_lines = SAMPLE_DATA.split('\n')
threading.Thread(target=create_file,
args=(tmpfilename, sample_lines, 0.25)).start()
time.sleep(0.05) # let the thread get started
# Read, and wait for lines to come
reader = LogfileReader(tmpfilename, tail=True)
for line in sample_lines:
self.assertEqual(line, reader.read())
def test_use_timestamps(self):
with tempfile.TemporaryDirectory() as tmpdirname:
logging.info('created temporary directory "%s"', tmpdirname)
tmpfilename = tmpdirname + '/mylog-2017-02-02'
sample_lines = SAMPLE_DATA.split('\n')
create_file(tmpfilename, sample_lines)
# Logs timestamps were created artificially with ~0.25 intervals
interval = 0.25
reader = LogfileReader(tmpfilename, use_timestamps=True)
then = 0
for line in sample_lines:
result = reader.read()
self.assertEqual(line, result)
now = time.time()
if then:
self.assertAlmostEqual(now - then, interval, places=1)
then = now
self.assertEqual(None, reader.read())
def __init__(self, port, filebase, instrument):
"""
```
port - UDP port on which to write records.
filebase - Prefix string to be matched (with a following "*") to fine
files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud
instrument - Instrument name prefix to add before sendind out on wire
```
"""
self.filebase = filebase
self.reader = LogfileReader(filebase=filebase, use_timestamps=True)
self.slice_n = SliceTransform(
fields='1:') # grab 2nd and subsequent fields
self.timestamp = TimestampTransform()
self.prefix = PrefixTransform(instrument)
self.writer = UDPWriter(port=port)
self.instrument = instrument
self.first_time = True
self.quit_flag = False
def __init__(self,
port,
filebase=None,
record_format=None,
time_format=TIME_FORMAT,
eol='\n'):
"""
```
port - UDP port on which to write records.
time_format - What format to use for timestamp
filebase - Prefix string to be matched (with a following "*") to find
files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud
record_format
If specified, a custom record format to use for extracting
timestamp and record. The default is '{timestamp:ti} {record}'
```
"""
self.port = port
self.time_format = time_format
self.filebase = filebase
self.record_format = record_format or '{timestamp:ti} {record}'
self.compiled_record_format = parse.compile(self.record_format)
# Do we have any files we can actually read from?
if not glob.glob(filebase + '*'):
logging.warning('No files matching "%s*"', filebase)
self.quit_flag = True
return
self.reader = LogfileReader(filebase=filebase,
use_timestamps=True,
record_format=self.record_format,
time_format=self.time_format)
self.writer = UDPWriter(port=port, eol=eol)
self.first_time = True
self.quit_flag = False
def run(self, loop=False):
"""Create the virtual port with socat and start feeding it records from
the designated logfile. If loop==True, loop when reaching end of input."""
self.socat_thread = threading.Thread(target=self._run_socat,
daemon=True)
self.socat_thread.start()
time.sleep(0.2)
self.reader = LogfileReader(filebase=self.source_file,
use_timestamps=self.use_timestamps,
time_format=self.time_format)
self.strip = SliceTransform('1:') # strip off the first field)
self.writer = TextFileWriter(self.write_port, truncate=True)
while not self.quit:
try:
record = self.reader.read() # get the next record
logging.debug('SimSerial got: %s', record)
# End of input? If loop==True, re-open the logfile from the start
if record is None:
if not loop:
break
self.reader = LogfileReader(
filebase=self.source_file,
use_timestamps=self.use_timestamps)
record = self.strip.transform(record) # strip the timestamp
if record:
logging.debug('SimSerial writing: %s', record)
self.writer.write(
record) # and write it to the virtual port
except (OSError, KeyboardInterrupt):
break
# If we're here, we got None from our input, and are done. Signal
# for run_socat to exit
self.quit = True
def test_basic_seek(self):
with tempfile.TemporaryDirectory() as tmpdirname:
logging.info('created temporary directory "%s"', tmpdirname)
filebase = tmpdirname + '/mylog-'
sample_lines = []
for f in sorted(SAMPLE_DATA_2):
create_file(filebase + f, SAMPLE_DATA_2[f])
sample_lines.extend(SAMPLE_DATA_2[f])
START_TIMESTAMP = get_msec_timestamp(sample_lines[0])
END_TIMESTAMP = get_msec_timestamp(sample_lines[-1])
reader = LogfileReader(filebase)
self.assertEqual(START_TIMESTAMP, reader.seek_time(0, 'start'))
self.assertEqual(sample_lines[0], reader.read())
self.assertEqual(START_TIMESTAMP + 1000,
reader.seek_time(1000, 'start'))
self.assertEqual(sample_lines[4], reader.read())
self.assertEqual(sample_lines[5], reader.read())
self.assertEqual(END_TIMESTAMP, reader.seek_time(0, 'end'))
self.assertEqual(None, reader.read())
class SimNetwork:
"""Open a network port and feed stored logfile data to it."""
############################
def __init__(self, port, filebase, instrument):
"""
```
port - UDP port on which to write records.
filebase - Prefix string to be matched (with a following "*") to fine
files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud
instrument - Instrument name prefix to add before sendind out on wire
```
"""
self.filebase = filebase
self.reader = LogfileReader(filebase=filebase, use_timestamps=True)
self.slice_n = SliceTransform(
fields='1:') # grab 2nd and subsequent fields
self.timestamp = TimestampTransform()
self.prefix = PrefixTransform(instrument)
self.writer = UDPWriter(port=port)
self.instrument = instrument
self.first_time = True
self.quit_flag = False
############################
def run(self, loop=False):
"""Start reading and writing data. If loop==True, loop when reaching
end of input.
"""
logging.info('Starting %s', self.instrument)
try:
while not self.quit_flag:
record = self.reader.read()
# If we don't have a record, we're (probably) at the end of
# the file. If it's the first time we've tried reading, it
# means we probably didn't get a usable file. Either break out
# (if we're not looping, or if we don't have a usable file),
# or start reading from the beginning (if we are looping and
# have a usable file).
if not record:
if not loop or self.first_time:
break
logging.info('Looping instrument %s', self.instrument)
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True)
continue
# Strip off timestamp and tack on a new one
record = self.slice_n.transform(record)
record = self.timestamp.transform(record)
# Add instrument name back on, and write to specified network
record = self.prefix.transform(record)
self.writer.write(record)
self.first_time = False
except (OSError, KeyboardInterrupt):
self.quit_flag = True
logging.info('Finished %s', self.instrument)
class SimSerial:
"""Create a virtual serial port and feed stored logfile data to it."""
############################
def __init__(self,
port,
source_file,
use_timestamps=True,
baudrate=9600,
bytesize=8,
parity='N',
stopbits=1,
timeout=None,
xonxoff=False,
rtscts=False,
write_timeout=None,
dsrdtr=False,
inter_byte_timeout=None,
exclusive=None):
"""Takes source file, whether to deliver data at rate indicated by
timestamps, and the standard parameters that a serial port takes."""
self.source_file = source_file
self.use_timestamps = use_timestamps
# We'll create two virtual ports: 'port' and 'port_in'; we will write
# to port_in and read the values back out from port
self.read_port = port
self.write_port = port + '_in'
self.serial_params = {
'baudrate': baudrate,
'byteside': bytesize,
'parity': parity,
'stopbits': stopbits,
'timeout': timeout,
'xonxoff': xonxoff,
'rtscts': rtscts,
'write_timeout': write_timeout,
'dsrdtr': dsrdtr,
'inter_byte_timeout': inter_byte_timeout,
'exclusive': exclusive
}
self.quit = False
# Finally, check that our needed 'socat' actually exists
if not subprocess.run(['which', 'socat'],
stdout=subprocess.PIPE).stdout:
raise NameError(
'Executable "socat" not found on path. Please refer '
'to installation guide to install socat.')
############################
def _run_socat(self):
"""Internal: run the actual command."""
verbose = '-d'
write_port_params = 'pty,link=%s,raw,echo=0' % self.write_port
read_port_params = 'pty,link=%s,raw,echo=0' % self.read_port
cmd = [
'/usr/bin/env',
'socat',
verbose,
#verbose, # repeating makes it more verbose
read_port_params,
write_port_params,
]
try:
# Run socat process using Popen, checking every second or so whether
# it's died (poll() != None) or we've gotten a quit signal.
logging.info('Calling: %s', ' '.join(cmd))
socat_process = subprocess.Popen(cmd)
while not self.quit and not socat_process.poll():
try:
socat_process.wait(1)
except subprocess.TimeoutExpired:
pass
except Exception as e:
logging.error('ERROR: socat command: %s', e)
# If here, process has terminated, or we've seen self.quit. We
# want both to be true: if we've terminated, set self.quit so that
# 'run' loop can exit. If self.quit, terminate process.
if self.quit:
socat_process.kill()
else:
self.quit = True
logging.info('Finished: %s', ' '.join(cmd))
############################
def run(self):
"""Create the virtual port with socat and start feeding it records from
the designated logfile."""
self.socat_thread = threading.Thread(target=self._run_socat)
self.socat_thread.start()
time.sleep(0.2)
self.reader = LogfileReader(filebase=self.source_file,
use_timestamps=self.use_timestamps)
self.strip = SliceTransform('1:') # strip off the first field)
self.writer = TextFileWriter(self.write_port, truncate=True)
while not self.quit:
record = self.reader.read() # get the next record
logging.debug('SimSerial got: %s', record)
if record is None:
break
record = self.strip.transform(record) # strip the timestamp
if record:
logging.debug('SimSerial writing: %s', record)
self.writer.write(record) # and write it to the virtual port
# If we're here, we got None from our input, and are done. Signal
# for run_socat to exit
self.quit = True
def test_read_time_range(self):
with tempfile.TemporaryDirectory() as tmpdirname:
logging.info('created temporary directory "%s"', tmpdirname)
filebase = tmpdirname + '/mylog-'
sample_lines = []
for f in sorted(SAMPLE_DATA_2):
create_file(filebase + f, SAMPLE_DATA_2[f])
sample_lines.extend(SAMPLE_DATA_2[f])
START_TIMESTAMP = get_msec_timestamp(sample_lines[0])
END_TIMESTAMP = get_msec_timestamp(sample_lines[-1])
reader = LogfileReader(filebase)
records = reader.read_time_range(START_TIMESTAMP, END_TIMESTAMP)
self.assertEqual(records, sample_lines[:-1])
records = reader.read_time_range(START_TIMESTAMP,
END_TIMESTAMP + .01)
self.assertEqual(records, sample_lines)
records = reader.read_time_range(START_TIMESTAMP + 1,
END_TIMESTAMP)
self.assertEqual(records, sample_lines[1:-1])
records = reader.read_time_range(START_TIMESTAMP + 1,
END_TIMESTAMP + 1)
self.assertEqual(records, sample_lines[1:])
records = reader.read_time_range(START_TIMESTAMP + .001, None)
self.assertEqual(records, sample_lines[1:])
records = reader.read_time_range(None, END_TIMESTAMP)
self.assertEqual(records, sample_lines[:-1])
records = reader.read_time_range(None, None)
self.assertEqual(records, sample_lines)
records = reader.read_time_range(START_TIMESTAMP, START_TIMESTAMP)
self.assertEqual(records, [])
records = reader.read_time_range(START_TIMESTAMP + 1000, None)
self.assertEqual(records, sample_lines[4:])
records = reader.read_time_range(START_TIMESTAMP + 1000,
END_TIMESTAMP - 1000)
self.assertEqual(records, sample_lines[4:9])
records = reader.read_time_range(START_TIMESTAMP + 1000,
START_TIMESTAMP + 500)
self.assertEqual(records, [])
with self.assertRaises(ValueError):
records = reader.read_time_range(START_TIMESTAMP - 1,
END_TIMESTAMP)
def test_seek_after_reading_to_end(self):
with tempfile.TemporaryDirectory() as tmpdirname:
logging.info('created temporary directory "%s"', tmpdirname)
filebase = tmpdirname + '/mylog-'
sample_lines = []
for f in sorted(SAMPLE_DATA_2):
create_file(filebase + f, SAMPLE_DATA_2[f])
sample_lines.extend(SAMPLE_DATA_2[f])
START_TIMESTAMP = get_msec_timestamp(sample_lines[0])
END_TIMESTAMP = get_msec_timestamp(sample_lines[-1])
reader = LogfileReader(filebase)
while reader.read() is not None:
pass
self.assertEqual(END_TIMESTAMP, reader.seek_time(0, 'current'))
self.assertEqual(None, reader.read())
self.assertEqual(END_TIMESTAMP, reader.seek_time(0, 'end'))
self.assertEqual(None, reader.read())
self.assertEqual(END_TIMESTAMP - 1000,
reader.seek_time(-1000, 'current'))
self.assertEqual(sample_lines[9], reader.read())
while reader.read() is not None:
pass
self.assertEqual(END_TIMESTAMP - 1000,
reader.seek_time(-1000, 'end'))
self.assertEqual(sample_lines[9], reader.read())
def test_seek_current(self):
with tempfile.TemporaryDirectory() as tmpdirname:
logging.info('created temporary directory "%s"', tmpdirname)
filebase = tmpdirname + '/mylog-'
sample_lines = []
for f in sorted(SAMPLE_DATA_2):
create_file(filebase + f, SAMPLE_DATA_2[f])
sample_lines.extend(SAMPLE_DATA_2[f])
START_TIMESTAMP = get_msec_timestamp(sample_lines[0])
END_TIMESTAMP = get_msec_timestamp(sample_lines[-1])
reader = LogfileReader(filebase)
self.assertEqual(START_TIMESTAMP, reader.seek_time(0, 'current'))
self.assertEqual(START_TIMESTAMP + 1000,
reader.seek_time(1000, 'current'))
# the first record with t >= START_TIMESTAMP + 1000 (= 1509840000441.672) is sample_lines[4]
timestamp_of_expected_next_record = get_msec_timestamp(
sample_lines[4]) # 1509840000460.595
self.assertEqual(timestamp_of_expected_next_record,
reader.seek_time(0, 'current'))
self.assertEqual(timestamp_of_expected_next_record - 500,
reader.seek_time(-500, 'current'))
# now the expected next record is sample_lines[3], since it's the first one with
# t > timestamp_of_expected_next_record - 500 (= 1509839999960.595)
self.assertEqual(sample_lines[3], reader.read())
self.assertEqual(sample_lines[4], reader.read())
# now seek to a time later than the last timestamp: check that the returned
# time is the requested time, and that a subsequent read() returns None
timestamp_of_expected_next_record = get_msec_timestamp(
sample_lines[5])
self.assertEqual(timestamp_of_expected_next_record + 10000,
reader.seek_time(10000, 'current'))
self.assertEqual(None, reader.read())
# check that 'current' time is now END_TIMESTAMP (= 1509840002486.203)
self.assertEqual(END_TIMESTAMP, reader.seek_time(0, 'current'))
# go back one second (to 1509840001486.203)
# next record should be sample_lines[9] (t = 1509840001726.024)
self.assertEqual(END_TIMESTAMP - 1000,
reader.seek_time(-1000, 'current'))
self.assertEqual(sample_lines[9], reader.read())
for filename in new_args.file.split(','):
readers.append(
TextFileReader(
file_spec=filename,
tail=all_args.tail,
refresh_file_spec=all_args.refresh_file_spec))
if new_args.network:
for addr in new_args.network.split(','):
readers.append(NetworkReader(network=addr))
if new_args.logfile:
for filebase in new_args.logfile.split(','):
readers.append(
LogfileReader(
filebase=filebase,
use_timestamps=all_args.logfile_use_timestamps,
refresh_file_spec=all_args.refresh_file_spec))
# For each comma-separated spec, parse out values for
# user@host:database:data_id[:message_type]. We count on
# --database_password having been specified somewhere.
if new_args.database:
password = all_args.database_password
(user, host_db) = new_args.database.split('@')
(host, database) = host_db.split(':', maxsplit=1)
if ':' in database:
(database, fields) = database.split(':')
else:
fields = None
readers.append(
DatabaseReader(fields=fields,
class SimSerial:
"""Create a virtual serial port and feed stored logfile data to it."""
############################
def __init__(self,
port,
time_format=TIME_FORMAT,
filebase=None,
record_format=None,
eol='\n',
baudrate=9600,
bytesize=8,
parity='N',
stopbits=1,
timeout=None,
xonxoff=False,
rtscts=False,
write_timeout=None,
dsrdtr=False,
inter_byte_timeout=None,
exclusive=None):
"""
Simulate a serial port, feeding it data from the specified file.
```
port - Temporary serial port to create and make available for reading
records.
filebase Possibly wildcarded string specifying files to be opened.
record_format
If specified, a custom record format to use for extracting
timestamp and record. The default is '{timestamp:ti} {record}'.
time_format - What format to use for timestamp
```
"""
# We'll create two virtual ports: 'port' and 'port_in'; we will write
# to port_in and read the values back out from port
self.read_port = port
self.write_port = port + '_in'
self.time_format = time_format
self.filebase = filebase
self.record_format = record_format or '{timestamp:ti} {record}'
self.compiled_record_format = parse.compile(self.record_format)
self.eol = eol
self.serial_params = None
# Complain, but go ahead if read_port or write_port exist.
for path in [self.read_port, self.write_port]:
if os.path.exists(path):
logging.warning('Path %s exists; overwriting!', path)
# Do we have any files we can actually read from?
if not glob.glob(filebase + '*'):
logging.warning('No files matching "%s*"', filebase)
return
# Set up our parameters
self.serial_params = {
'baudrate': baudrate,
'byteside': bytesize,
'parity': parity,
'stopbits': stopbits,
'timeout': timeout,
'xonxoff': xonxoff,
'rtscts': rtscts,
'write_timeout': write_timeout,
'dsrdtr': dsrdtr,
'inter_byte_timeout': inter_byte_timeout,
'exclusive': exclusive
}
self.quit = False
############################
def run(self, loop=False):
# If self.serial_params is None, it means that either read or
# write device already exist, so we shouldn't actually run, or
# we'll destroy them.
if not self.serial_params:
return
try:
# Create simulated serial port (something like /dev/ttys2), and link
# it to the port they want to connect to (like /tmp/tty_s330).
write_fd, read_fd = pty.openpty() # open the pseudoterminal
true_read_port = os.ttyname(
read_fd) # this is the true filename of port
# Get rid of any previous symlink if it exists, and symlink the new pty
try:
os.unlink(self.read_port)
except FileNotFoundError:
pass
os.symlink(true_read_port, self.read_port)
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True,
record_format=self.record_format,
time_format=self.time_format)
logging.info('Starting %s: %s', self.read_port, self.filebase)
while not self.quit:
try:
record = self.reader.read() # get the next record
logging.debug('SimSerial got: %s', record)
# End of input? If loop==True, re-open the logfile from the start
if record is None:
if not loop:
break
self.reader = LogfileReader(
filebase=self.filebase,
use_timestamps=True,
record_format=self.record_format,
time_format=self.time_format)
# We've now got a record. Try parsing timestamp off it
try:
parsed_record = self.compiled_record_format.parse(
record).named
record = parsed_record['record']
# We had a problem parsing. Discard record and try reading next one.
except (KeyError, ValueError, TypeError, AttributeError):
logging.warning('Unable to parse record into "%s"',
self.record_format)
logging.warning('Record: %s', record)
continue
if not record:
continue
logging.debug('SimSerial writing: %s', record)
os.write(write_fd, (record + self.eol).encode('utf8'))
except (OSError, KeyboardInterrupt):
break
# If we're here, we got None from our input, and are done. Signal
# for run_socat to exit
self.quit = True
finally:
# Get rid of the symlink we've created
os.unlink(self.read_port)
class SimSerial:
"""Create a virtual serial port and feed stored logfile data to it."""
############################
def __init__(self, port, prefix=None, timestamp=False,
time_format=TIME_FORMAT, filebase=None, eol='\n',
baudrate=9600, bytesize=8, parity='N', stopbits=1,
timeout=None, xonxoff=False, rtscts=False, write_timeout=None,
dsrdtr=False, inter_byte_timeout=None, exclusive=None):
"""
Simulate a serial port, feeding it data from the specified file.
```
port - Temporary serial port to create and make available for reading
records.
prefix - If non-empty, prefix name prefix to add
timestamp = If True, apply current timestamp to record
time_format - What format to use for timestamp
```
"""
# We'll create two virtual ports: 'port' and 'port_in'; we will write
# to port_in and read the values back out from port
self.read_port = port
self.write_port = port + '_in'
self.prefix = PrefixTransform(prefix) if prefix else None
self.timestamp = TimestampTransform() if timestamp else None
self.time_format = time_format
self.filebase = filebase
self.serial_params = None
# Complain, but go ahead if read_port or write_port exist.
for path in [self.read_port, self.write_port]:
if os.path.exists(path):
logging.warning('Path %s exists; overwriting!')
# Do we have any files we can actually read from?
if not glob.glob(filebase + '*'):
logging.warning('%s: no files matching "%s*"', prefix, filebase)
return
# Set up our parameters
self.serial_params = {'baudrate': baudrate,
'byteside': bytesize,
'parity': parity,
'stopbits': stopbits,
'timeout': timeout,
'xonxoff': xonxoff,
'rtscts': rtscts,
'write_timeout': write_timeout,
'dsrdtr': dsrdtr,
'inter_byte_timeout': inter_byte_timeout,
'exclusive': exclusive}
self.quit = False
# Finally, find path to socat executable
self.socat_path = None
for socat_path in ['/usr/bin/socat', '/usr/local/bin/socat']:
if os.path.exists(socat_path) and os.path.isfile(socat_path):
self.socat_path = socat_path
if not self.socat_path:
raise NameError('Executable "socat" not found on path. Please refer '
'to installation guide to install socat.')
############################
def _run_socat(self):
"""Internal: run the actual command."""
verbose = '-d'
write_port_params = 'pty,link=%s,raw,echo=0' % self.write_port
read_port_params = 'pty,link=%s,raw,echo=0' % self.read_port
cmd = [self.socat_path,
verbose,
#verbose, # repeating makes it more verbose
read_port_params,
write_port_params,
]
try:
# Run socat process using Popen, checking every second or so whether
# it's died (poll() != None) or we've gotten a quit signal.
logging.info('Calling: %s', ' '.join(cmd))
socat_process = subprocess.Popen(cmd)
while not self.quit and not socat_process.poll():
try:
socat_process.wait(1)
except subprocess.TimeoutExpired:
pass
except Exception as e:
logging.error('ERROR: socat command: %s', e)
# If here, process has terminated, or we've seen self.quit. We
# want both to be true: if we've terminated, set self.quit so that
# 'run' loop can exit. If self.quit, terminate process.
if self.quit:
socat_process.kill()
else:
self.quit = True
logging.info('Finished: %s', ' '.join(cmd))
############################
def run(self, loop=False):
# If self.serial_params is None, it means that either read or
# write device already exist, so we shouldn't actually run, or
# we'll destroy them.
if not self.serial_params:
return
"""Create the virtual port with socat and start feeding it records from
the designated logfile. If loop==True, loop when reaching end of input."""
self.socat_thread = threading.Thread(target=self._run_socat, daemon=True)
self.socat_thread.start()
time.sleep(0.2)
self.reader = LogfileReader(filebase=self.filebase, use_timestamps=True,
time_format=self.time_format)
self.slice_n = SliceTransform('1:') # strip off the first field (timestamp)
self.writer = TextFileWriter(self.write_port, truncate=True)
logging.info('Starting %s: %s', self.read_port, self.filebase)
while not self.quit:
try:
record = self.reader.read() # get the next record
logging.debug('SimSerial got: %s', record)
# End of input? If loop==True, re-open the logfile from the start
if record is None:
if not loop:
break
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True,
time_format=self.time_format)
record = self.slice_n.transform(record) # strip the timestamp
if not record:
continue
if self.timestamp: # do we want to add a timestamp?
record = self.timestamp.transform(record)
if self.prefix: # do we want to add prefix?
record = self.prefix.transform(record)
logging.debug('SimSerial writing: %s', record)
self.writer.write(record) # and write it to the virtual port
except (OSError, KeyboardInterrupt):
break
# If we're here, we got None from our input, and are done. Signal
# for run_socat to exit
self.quit = True
def run(self, loop=False):
# If self.serial_params is None, it means that either read or
# write device already exist, so we shouldn't actually run, or
# we'll destroy them.
if not self.serial_params:
return
try:
# Create simulated serial port (something like /dev/ttys2), and link
# it to the port they want to connect to (like /tmp/tty_s330).
write_fd, read_fd = pty.openpty() # open the pseudoterminal
true_read_port = os.ttyname(
read_fd) # this is the true filename of port
# Get rid of any previous symlink if it exists, and symlink the new pty
try:
os.unlink(self.read_port)
except FileNotFoundError:
pass
os.symlink(true_read_port, self.read_port)
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True,
record_format=self.record_format,
time_format=self.time_format)
logging.info('Starting %s: %s', self.read_port, self.filebase)
while not self.quit:
try:
record = self.reader.read() # get the next record
logging.debug('SimSerial got: %s', record)
# End of input? If loop==True, re-open the logfile from the start
if record is None:
if not loop:
break
self.reader = LogfileReader(
filebase=self.filebase,
use_timestamps=True,
record_format=self.record_format,
time_format=self.time_format)
# We've now got a record. Try parsing timestamp off it
try:
parsed_record = self.compiled_record_format.parse(
record).named
record = parsed_record['record']
# We had a problem parsing. Discard record and try reading next one.
except (KeyError, ValueError, TypeError, AttributeError):
logging.warning('Unable to parse record into "%s"',
self.record_format)
logging.warning('Record: %s', record)
continue
if not record:
continue
logging.debug('SimSerial writing: %s', record)
os.write(write_fd, (record + self.eol).encode('utf8'))
except (OSError, KeyboardInterrupt):
break
# If we're here, we got None from our input, and are done. Signal
# for run_socat to exit
self.quit = True
finally:
# Get rid of the symlink we've created
os.unlink(self.read_port)
class SimUDP:
"""Open a network port and feed stored logfile data to it."""
############################
def __init__(self,
port,
filebase=None,
record_format=None,
time_format=TIME_FORMAT,
eol='\n'):
"""
```
port - UDP port on which to write records.
time_format - What format to use for timestamp
filebase - Prefix string to be matched (with a following "*") to find
files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud
record_format
If specified, a custom record format to use for extracting
timestamp and record. The default is '{timestamp:ti} {record}'
```
"""
self.port = port
self.time_format = time_format
self.filebase = filebase
self.record_format = record_format or '{timestamp:ti} {record}'
self.compiled_record_format = parse.compile(self.record_format)
# Do we have any files we can actually read from?
if not glob.glob(filebase + '*'):
logging.warning('No files matching "%s*"', filebase)
self.quit_flag = True
return
self.reader = LogfileReader(filebase=filebase,
use_timestamps=True,
record_format=self.record_format,
time_format=self.time_format)
self.writer = UDPWriter(port=port, eol=eol)
self.first_time = True
self.quit_flag = False
############################
def run(self, loop=False):
"""Start reading and writing data. If loop==True, loop when reaching
end of input.
"""
logging.info('Starting %s: %s', self.port, self.filebase)
try:
while not self.quit_flag:
record = self.reader.read()
# If we don't have a record, we're (probably) at the end of
# the file. If it's the first time we've tried reading, it
# means we probably didn't get a usable file. Either break out
# (if we're not looping, or if we don't have a usable file),
# or start reading from the beginning (if we are looping and
# have a usable file).
if not record:
if not loop or self.first_time:
break
logging.info('Looping instrument %s', self.prefix)
self.reader = LogfileReader(
filebase=self.filebase,
record_format=self.record_format,
use_timestamps=True)
continue
# We've now got a record. Try parsing timestamp off it
try:
parsed_record = self.compiled_record_format.parse(
record).named
record = parsed_record['record']
# We had a problem parsing. Discard record and try reading next one.
except (KeyError, ValueError, AttributeError):
logging.warning('Unable to parse record into "%s"',
self.record_format)
logging.warning('Record: %s', record)
continue
if not record:
continue
self.writer.write(record)
self.first_time = False
except (OSError, KeyboardInterrupt):
self.quit_flag = True
logging.info('Finished %s', self.prefix)
class SimUDP:
"""Open a network port and feed stored logfile data to it."""
############################
def __init__(self, port, prefix=None, timestamp=False,
time_format=TIME_FORMAT, filebase=None, eol='\n'):
"""
```
port - UDP port on which to write records.
prefix - If non-empty, prefix to add
timestamp = If True, apply current timestamp to record
time_format - What format to use for timestamp
filebase - Prefix string to be matched (with a following "*") to find
files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud
```
"""
self.port = port
self.prefix = PrefixTransform(prefix) if prefix else None
self.timestamp = TimestampTransform() if timestamp else None
self.time_format = time_format
self.filebase = filebase
# Do we have any files we can actually read from?
if not glob.glob(filebase + '*'):
logging.warning('No files matching "%s*"', filebase)
self.quit_flag = True
return
self.reader = LogfileReader(filebase=filebase, use_timestamps=True,
time_format=self.time_format)
self.slice_n = SliceTransform(fields='1:') # strip off timestamp
self.writer = UDPWriter(port=port, eol=eol)
self.first_time = True
self.quit_flag = False
############################
def run(self, loop=False):
"""Start reading and writing data. If loop==True, loop when reaching
end of input.
"""
logging.info('Starting %s: %s', self.port, self.filebase)
try:
while not self.quit_flag:
record = self.reader.read()
# If we don't have a record, we're (probably) at the end of
# the file. If it's the first time we've tried reading, it
# means we probably didn't get a usable file. Either break out
# (if we're not looping, or if we don't have a usable file),
# or start reading from the beginning (if we are looping and
# have a usable file).
if not record:
if not loop or self.first_time:
break
logging.info('Looping instrument %s', self.prefix)
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True)
continue
# Strip off timestamp
record = self.slice_n.transform(record)
if not record:
continue
record = record.strip()
if not record:
continue
if self.timestamp: # do we want to add a timestamp?
record = self.timestamp.transform(record)
if self.prefix: # do we want to add prefix?
record = self.prefix.transform(record)
self.writer.write(record)
self.first_time = False
except (OSError, KeyboardInterrupt):
self.quit_flag = True
logging.info('Finished %s', self.prefix)
port = int(addr[-1]) # port is last arg
if len(addr) > 1:
source = addr[-2] # source (multi/broadcast) is prev arg
if len(addr) > 2:
parser.error('Format error for --udp argument. Format '
'should be [source:]port')
readers.append(UDPReader(port=port, source=source, eol=eol))
if new_args.redis:
for channel in new_args.redis.split(','):
readers.append(RedisReader(channel=channel))
if new_args.logfile:
for filebase in new_args.logfile.split(','):
readers.append(LogfileReader(
filebase=filebase, use_timestamps=all_args.logfile_use_timestamps,
time_format=all_args.time_format,
refresh_file_spec=all_args.refresh_file_spec))
if new_args.cached_data_server:
fields = new_args.cached_data_server
server = None
if fields.find('@') > 0:
fields, server = fields.split('@')
subscription = {'fields': {f:{'seconds':0} for f in fields.split(',')}}
if server:
readers.append(CachedDataReader(subscription=subscription,
data_server=server))
else:
readers.append(CachedDataReader(subscription=subscription))
# For each comma-separated spec, parse out values for