def __init__(self, port, prefix=None, timestamp=False,
time_format=TIME_FORMAT, filebase=None, eol='\n'):
"""
```
port - UDP port on which to write records.
prefix - If non-empty, prefix to add
timestamp = If True, apply current timestamp to record
time_format - What format to use for timestamp
filebase - Prefix string to be matched (with a following "*") to find
files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud
```
"""
self.port = port
self.prefix = PrefixTransform(prefix) if prefix else None
self.timestamp = TimestampTransform() if timestamp else None
self.time_format = time_format
self.filebase = filebase
# Do we have any files we can actually read from?
if not glob.glob(filebase + '*'):
logging.warning('No files matching "%s*"', filebase)
self.quit_flag = True
return
self.reader = LogfileReader(filebase=filebase, use_timestamps=True,
time_format=self.time_format)
self.slice_n = SliceTransform(fields='1:') # strip off timestamp
self.writer = UDPWriter(port=port, eol=eol)
self.first_time = True
self.quit_flag = False
def test_custom(self):
transform = TimestampTransform(time_format=timestamp.DATE_FORMAT)
self.assertIsNone(transform.transform(None))
result = transform.transform('blah')
today = timestamp.date_str()
self.assertEqual(result.split()[0], today)
self.assertEqual(result.split()[1], 'blah')
def test_default(self):
transform = TimestampTransform()
self.assertIsNone(transform.transform(None))
result = transform.transform('blah')
time_str = result.split()[0]
then = timestamp.timestamp(time_str=time_str)
now = timestamp.timestamp()
self.assertAlmostEqual(then, now, places=1)
self.assertEqual(result.split()[1], 'blah')
def test_list(self):
transform = TimestampTransform()
self.assertIsNone(transform.transform(None))
record = ['foo', 'bar', 'baz']
result = transform.transform(record)
timestamps = [r.split()[0] for r in result]
self.assertEqual(timestamps[0], timestamps[1])
self.assertEqual(timestamps[1], timestamps[2])
then = timestamp.timestamp(time_str=timestamps[0])
now = timestamp.timestamp()
self.assertAlmostEqual(then, now, places=1)
sources = [r.split()[1] for r in result]
self.assertEqual(sources, record)
def __init__(self, port, filebase, instrument):
"""
```
port - UDP port on which to write records.
filebase - Prefix string to be matched (with a following "*") to fine
files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud
instrument - Instrument name prefix to add before sendind out on wire
```
"""
self.filebase = filebase
self.reader = LogfileReader(filebase=filebase, use_timestamps=True)
self.slice_n = SliceTransform(
fields='1:') # grab 2nd and subsequent fields
self.timestamp = TimestampTransform()
self.prefix = PrefixTransform(instrument)
self.writer = UDPWriter(port=port)
self.instrument = instrument
self.first_time = True
self.quit_flag = False
class SimNetwork:
"""Open a network port and feed stored logfile data to it."""
############################
def __init__(self, port, filebase, instrument):
"""
```
port - UDP port on which to write records.
filebase - Prefix string to be matched (with a following "*") to fine
files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud
instrument - Instrument name prefix to add before sendind out on wire
```
"""
self.filebase = filebase
self.reader = LogfileReader(filebase=filebase, use_timestamps=True)
self.slice_n = SliceTransform(
fields='1:') # grab 2nd and subsequent fields
self.timestamp = TimestampTransform()
self.prefix = PrefixTransform(instrument)
self.writer = UDPWriter(port=port)
self.instrument = instrument
self.first_time = True
self.quit_flag = False
############################
def run(self, loop=False):
"""Start reading and writing data. If loop==True, loop when reaching
end of input.
"""
logging.info('Starting %s', self.instrument)
try:
while not self.quit_flag:
record = self.reader.read()
# If we don't have a record, we're (probably) at the end of
# the file. If it's the first time we've tried reading, it
# means we probably didn't get a usable file. Either break out
# (if we're not looping, or if we don't have a usable file),
# or start reading from the beginning (if we are looping and
# have a usable file).
if not record:
if not loop or self.first_time:
break
logging.info('Looping instrument %s', self.instrument)
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True)
continue
# Strip off timestamp and tack on a new one
record = self.slice_n.transform(record)
record = self.timestamp.transform(record)
# Add instrument name back on, and write to specified network
record = self.prefix.transform(record)
self.writer.write(record)
self.first_time = False
except (OSError, KeyboardInterrupt):
self.quit_flag = True
logging.info('Finished %s', self.instrument)
def __init__(self, port, prefix=None, timestamp=False,
time_format=TIME_FORMAT, filebase=None, eol='\n',
baudrate=9600, bytesize=8, parity='N', stopbits=1,
timeout=None, xonxoff=False, rtscts=False, write_timeout=None,
dsrdtr=False, inter_byte_timeout=None, exclusive=None):
"""
Simulate a serial port, feeding it data from the specified file.
```
port - Temporary serial port to create and make available for reading
records.
prefix - If non-empty, prefix name prefix to add
timestamp = If True, apply current timestamp to record
time_format - What format to use for timestamp
```
"""
# We'll create two virtual ports: 'port' and 'port_in'; we will write
# to port_in and read the values back out from port
self.read_port = port
self.write_port = port + '_in'
self.prefix = PrefixTransform(prefix) if prefix else None
self.timestamp = TimestampTransform() if timestamp else None
self.time_format = time_format
self.filebase = filebase
self.serial_params = None
# Complain, but go ahead if read_port or write_port exist.
for path in [self.read_port, self.write_port]:
if os.path.exists(path):
logging.warning('Path %s exists; overwriting!')
# Do we have any files we can actually read from?
if not glob.glob(filebase + '*'):
logging.warning('%s: no files matching "%s*"', prefix, filebase)
return
# Set up our parameters
self.serial_params = {'baudrate': baudrate,
'byteside': bytesize,
'parity': parity,
'stopbits': stopbits,
'timeout': timeout,
'xonxoff': xonxoff,
'rtscts': rtscts,
'write_timeout': write_timeout,
'dsrdtr': dsrdtr,
'inter_byte_timeout': inter_byte_timeout,
'exclusive': exclusive}
self.quit = False
# Finally, find path to socat executable
self.socat_path = None
for socat_path in ['/usr/bin/socat', '/usr/local/bin/socat']:
if os.path.exists(socat_path) and os.path.isfile(socat_path):
self.socat_path = socat_path
if not self.socat_path:
raise NameError('Executable "socat" not found on path. Please refer '
'to installation guide to install socat.')
class SimSerial:
"""Create a virtual serial port and feed stored logfile data to it."""
############################
def __init__(self, port, prefix=None, timestamp=False,
time_format=TIME_FORMAT, filebase=None, eol='\n',
baudrate=9600, bytesize=8, parity='N', stopbits=1,
timeout=None, xonxoff=False, rtscts=False, write_timeout=None,
dsrdtr=False, inter_byte_timeout=None, exclusive=None):
"""
Simulate a serial port, feeding it data from the specified file.
```
port - Temporary serial port to create and make available for reading
records.
prefix - If non-empty, prefix name prefix to add
timestamp = If True, apply current timestamp to record
time_format - What format to use for timestamp
```
"""
# We'll create two virtual ports: 'port' and 'port_in'; we will write
# to port_in and read the values back out from port
self.read_port = port
self.write_port = port + '_in'
self.prefix = PrefixTransform(prefix) if prefix else None
self.timestamp = TimestampTransform() if timestamp else None
self.time_format = time_format
self.filebase = filebase
self.serial_params = None
# Complain, but go ahead if read_port or write_port exist.
for path in [self.read_port, self.write_port]:
if os.path.exists(path):
logging.warning('Path %s exists; overwriting!')
# Do we have any files we can actually read from?
if not glob.glob(filebase + '*'):
logging.warning('%s: no files matching "%s*"', prefix, filebase)
return
# Set up our parameters
self.serial_params = {'baudrate': baudrate,
'byteside': bytesize,
'parity': parity,
'stopbits': stopbits,
'timeout': timeout,
'xonxoff': xonxoff,
'rtscts': rtscts,
'write_timeout': write_timeout,
'dsrdtr': dsrdtr,
'inter_byte_timeout': inter_byte_timeout,
'exclusive': exclusive}
self.quit = False
# Finally, find path to socat executable
self.socat_path = None
for socat_path in ['/usr/bin/socat', '/usr/local/bin/socat']:
if os.path.exists(socat_path) and os.path.isfile(socat_path):
self.socat_path = socat_path
if not self.socat_path:
raise NameError('Executable "socat" not found on path. Please refer '
'to installation guide to install socat.')
############################
def _run_socat(self):
"""Internal: run the actual command."""
verbose = '-d'
write_port_params = 'pty,link=%s,raw,echo=0' % self.write_port
read_port_params = 'pty,link=%s,raw,echo=0' % self.read_port
cmd = [self.socat_path,
verbose,
#verbose, # repeating makes it more verbose
read_port_params,
write_port_params,
]
try:
# Run socat process using Popen, checking every second or so whether
# it's died (poll() != None) or we've gotten a quit signal.
logging.info('Calling: %s', ' '.join(cmd))
socat_process = subprocess.Popen(cmd)
while not self.quit and not socat_process.poll():
try:
socat_process.wait(1)
except subprocess.TimeoutExpired:
pass
except Exception as e:
logging.error('ERROR: socat command: %s', e)
# If here, process has terminated, or we've seen self.quit. We
# want both to be true: if we've terminated, set self.quit so that
# 'run' loop can exit. If self.quit, terminate process.
if self.quit:
socat_process.kill()
else:
self.quit = True
logging.info('Finished: %s', ' '.join(cmd))
############################
def run(self, loop=False):
# If self.serial_params is None, it means that either read or
# write device already exist, so we shouldn't actually run, or
# we'll destroy them.
if not self.serial_params:
return
"""Create the virtual port with socat and start feeding it records from
the designated logfile. If loop==True, loop when reaching end of input."""
self.socat_thread = threading.Thread(target=self._run_socat, daemon=True)
self.socat_thread.start()
time.sleep(0.2)
self.reader = LogfileReader(filebase=self.filebase, use_timestamps=True,
time_format=self.time_format)
self.slice_n = SliceTransform('1:') # strip off the first field (timestamp)
self.writer = TextFileWriter(self.write_port, truncate=True)
logging.info('Starting %s: %s', self.read_port, self.filebase)
while not self.quit:
try:
record = self.reader.read() # get the next record
logging.debug('SimSerial got: %s', record)
# End of input? If loop==True, re-open the logfile from the start
if record is None:
if not loop:
break
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True,
time_format=self.time_format)
record = self.slice_n.transform(record) # strip the timestamp
if not record:
continue
if self.timestamp: # do we want to add a timestamp?
record = self.timestamp.transform(record)
if self.prefix: # do we want to add prefix?
record = self.prefix.transform(record)
logging.debug('SimSerial writing: %s', record)
self.writer.write(record) # and write it to the virtual port
except (OSError, KeyboardInterrupt):
break
# If we're here, we got None from our input, and are done. Signal
# for run_socat to exit
self.quit = True
class SimUDP:
"""Open a network port and feed stored logfile data to it."""
############################
def __init__(self, port, prefix=None, timestamp=False,
time_format=TIME_FORMAT, filebase=None, eol='\n'):
"""
```
port - UDP port on which to write records.
prefix - If non-empty, prefix to add
timestamp = If True, apply current timestamp to record
time_format - What format to use for timestamp
filebase - Prefix string to be matched (with a following "*") to find
files to be used. e.g. /tmp/log/NBP1406/knud/raw/NBP1406_knud
```
"""
self.port = port
self.prefix = PrefixTransform(prefix) if prefix else None
self.timestamp = TimestampTransform() if timestamp else None
self.time_format = time_format
self.filebase = filebase
# Do we have any files we can actually read from?
if not glob.glob(filebase + '*'):
logging.warning('No files matching "%s*"', filebase)
self.quit_flag = True
return
self.reader = LogfileReader(filebase=filebase, use_timestamps=True,
time_format=self.time_format)
self.slice_n = SliceTransform(fields='1:') # strip off timestamp
self.writer = UDPWriter(port=port, eol=eol)
self.first_time = True
self.quit_flag = False
############################
def run(self, loop=False):
"""Start reading and writing data. If loop==True, loop when reaching
end of input.
"""
logging.info('Starting %s: %s', self.port, self.filebase)
try:
while not self.quit_flag:
record = self.reader.read()
# If we don't have a record, we're (probably) at the end of
# the file. If it's the first time we've tried reading, it
# means we probably didn't get a usable file. Either break out
# (if we're not looping, or if we don't have a usable file),
# or start reading from the beginning (if we are looping and
# have a usable file).
if not record:
if not loop or self.first_time:
break
logging.info('Looping instrument %s', self.prefix)
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True)
continue
# Strip off timestamp
record = self.slice_n.transform(record)
if not record:
continue
record = record.strip()
if not record:
continue
if self.timestamp: # do we want to add a timestamp?
record = self.timestamp.transform(record)
if self.prefix: # do we want to add prefix?
record = self.prefix.transform(record)
self.writer.write(record)
self.first_time = False
except (OSError, KeyboardInterrupt):
self.quit_flag = True
logging.info('Finished %s', self.prefix)
LOGGING_FORMAT = '%(asctime)-15s %(message)s'
logging.basicConfig(format=LOGGING_FORMAT)
LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG}
args.verbosity = min(args.verbosity, max(LOG_LEVELS))
logging.getLogger().setLevel(LOG_LEVELS[args.verbosity])
reader = TextFileReader(file_spec=args.read, tail=args.tail)
writer = TextFileWriter(filename=args.write)
if args.prefix:
prefix_transform = PrefixTransform(args.prefix)
if args.timestamp:
timestamp_transform = TimestampTransform()
while True:
record = reader.read()
now = time.time()
logging.info('Got record: %s', record)
if record is None:
if not args.tail:
break
else:
if args.timestamp:
record = timestamp_transform.transform(record)
if args.prefix:
# because it can take so many parameters. Use the kwargs trick to
# pass them all in.
if new_args.serial:
kwargs = {}
for pair in new_args.serial.split(','):
(key, value) = pair.split('=')
kwargs[key] = value
readers.append(SerialReader(**kwargs))
##########################
# Transforms
if new_args.slice:
transforms.append(SliceTransform(new_args.slice,
all_args.slice_separator))
if new_args.timestamp:
transforms.append(TimestampTransform(time_format=all_args.time_format))
if new_args.prefix:
transforms.append(PrefixTransform(new_args.prefix))
if new_args.extract:
transforms.append(ExtractFieldTransform(new_args.extract))
if new_args.regex_filter:
transforms.append(RegexFilterTransform(new_args.regex_filter))
if new_args.qc_filter:
transforms.append(QCFilterTransform(new_args.qc_filter))
if new_args.parse_nmea:
transforms.append(
ParseNMEATransform(
message_path=all_args.parse_nmea_message_path,
sensor_path=all_args.parse_nmea_sensor_path,
sensor_model_path=all_args.parse_nmea_sensor_model_path,
time_format=all_args.time_format)
# pass them all in.
if new_args.serial:
kwargs = {}
for pair in new_args.serial.split(','):
(key, value) = pair.split('=')
kwargs[key] = value
readers.append(SerialReader(**kwargs))
##########################
# Transforms
if new_args.slice:
transforms.append(
SliceTransform(new_args.slice, all_args.slice_separator))
if new_args.timestamp:
transforms.append(
TimestampTransform(time_format=all_args.time_format))
if new_args.prefix:
transforms.append(PrefixTransform(new_args.prefix))
if new_args.regex_filter:
transforms.append(RegexFilterTransform(new_args.regex_filter))
if new_args.qc_filter:
transforms.append(QCFilterTransform(new_args.qc_filter))
if new_args.parse_nmea:
transforms.append(
ParseNMEATransform(time_format=all_args.time_format))
if new_args.aggregate_xml:
transforms.append(
XMLAggregatorTransform(new_args.aggregate_xml))
##########################
# Writers
# because it can take so many parameters. Use the kwargs trick to
# pass them all in.
if new_args.serial:
kwargs = {}
for pair in new_args.serial.split(','):
(key, value) = pair.split('=')
kwargs[key] = value
readers.append(SerialReader(**kwargs))
##########################
# Transforms
if new_args.slice:
transforms.append(
SliceTransform(new_args.slice, all_args.slice_separator))
if new_args.timestamp:
transforms.append(TimestampTransform())
if new_args.prefix:
transforms.append(PrefixTransform(new_args.prefix))
if new_args.regex_filter:
transforms.append(RegexFilterTransform(new_args.regex_filter))
if new_args.qc_filter:
transforms.append(QCFilterTransform(new_args.qc_filter))
if new_args.parse_nmea:
transforms.append(ParseNMEATransform())
##########################
# Writers
if new_args.write_file:
for filename in new_args.write_file.split(','):
if filename == '-':
filename = None
