def __init__(self,
readers=[],
transforms=[],
writers=[],
stderr_writers=[],
host_id='',
interval=0,
name=None,
check_format=False):
"""listener = Listener(readers, transforms=[], writers=[],
interval=0, check_format=False)
readers A single Reader or a list of Readers.
transforms A single Transform or a list of zero or more Transforms
writers A single Writer or a list of zero or more Writers
stderr_writers A single Writer or a list of zero or more Writers to which
the logger's stderr should be written.
interval How long to sleep before reading sequential records
name Optional human-readable short name to be used in displays
check_format If True, attempt to check that Reader/Transform/Writer
formats are compatible, and throw a ValueError if they
are not. If check_format is False (the default) the
output_format() of the whole reader will be
formats.Unknown.
Sample use:
listener = Listener(readers=[NetworkReader(':6221'),
NetworkReader(':6223')],
transforms=[TimestampTransform()],
writers=[TextFileWriter('/logs/network_recs'),
TextFileWriter(None)],
interval=0.2)
listener.run()
Calling listener.quit() from another thread will cause the run() loop
to exit.
"""
logging.info('Instantiating %s logger', name or 'unnamed')
###########
# Create readers, writers, etc.
self.reader = ComposedReader(readers=readers,
check_format=check_format)
self.writer = ComposedWriter(transforms=transforms,
writers=writers,
check_format=check_format)
self.interval = interval
self.name = name or 'Unnamed listener'
self.last_read = 0
self.quit_signalled = False
def __init__(self,
readers,
transforms=[],
writers=[],
host_id='',
interval=0,
name=None,
check_format=False):
"""
listener = Listener(readers, transforms=[], writers=[],
interval=0, check_format=False)
readers A single Reader or a list of Readers.
transforms A single Transform or a list of zero or more Transforms
writers A single Writer or a list of zero or more Writers
host_id Optional host_id on which Listener is to be run. Ignored
here, but it may show up as part of a config, so we need
to be able to handle is in kwargs.
interval How long to sleep before reading sequential records
name Optional human-readable short name to be used in displays
check_format If True, attempt to check that Reader/Transform/Writer
formats are compatible, and throw a ValueError if they
are not. If check_format is False (the default) the
output_format() of the whole reader will be
formats.Unknown.
Sample use:
listener = Listener(readers=[NetworkReader(':6221'),
NetworkReader(':6223')],
transforms=[TimestampTransform()],
writers=[TextFileWriter('/logs/network_recs'),
TextFileWriter(None)],
interval=0.2)
listener.run()
Calling listener.quit() from another thread will cause the run() loop
to exit.
"""
self.reader = ComposedReader(readers=readers,
check_format=check_format)
self.writer = ComposedWriter(transforms=transforms,
writers=writers,
check_format=check_format)
self.interval = interval
self.name = name or 'Unnamed listener'
self.last_read = 0
self.quit_signalled = False
def test_check_format(self):
# This should be okay
ComposedReader([
TextFileReader(self.tmpfilenames[0]),
TextFileReader(self.tmpfilenames[1])
],
check_format=True)
# This should not be - no common reader format
with self.assertRaises(ValueError):
ComposedReader([TextFileReader(self.tmpfilenames[0]),
Reader()],
check_format=True)
def test_all_files(self):
# Use TextFileReader's 'interval' flag to make sure we interleave
# reads the way we expect. Also make sure transforms get applied
# in proper order.
readers = []
for tmpfilename in self.tmpfilenames:
readers.append(TextFileReader(tmpfilename, interval=0.2))
#readers.append(TextFileReader()) # read from stdin
prefix_1 = PrefixTransform('prefix_1')
prefix_2 = PrefixTransform('prefix_2')
reader = ComposedReader(readers, [prefix_1, prefix_2])
# Clunkly quick way of slicing lines
i = 0
expected_lines = []
while True:
next_lines = []
for f in sorted(SAMPLE_DATA):
if i < len(SAMPLE_DATA[f]):
line = 'prefix_2 prefix_1 ' + SAMPLE_DATA[f][i]
next_lines.append(line)
if next_lines:
expected_lines.append(next_lines)
i += 1
else:
break
logging.debug('Expected lines %s', expected_lines)
# Next line from each of the files can come in arbitrary order,
# but within the file, lines should arrive in order, and we
# should receive first line from each file before we receive
# next line from any of them.
while expected_lines:
next_lines = expected_lines.pop(0)
while next_lines:
record = reader.read()
logging.info('read: %s; expected one of: %s', record,
next_lines)
self.assertTrue(record in next_lines)
if record in next_lines:
next_lines.remove(record)
self.assertEqual(None, reader.read())
class Listener:
"""Listener is a simple, yet relatively self-contained class that
takes a list of one or more Readers, a list of zero or more
Transforms, and a list of zero or more Writers. It calls the Readers
(in parallel) to acquire records, passes those records through the
Transforms (in series), and sends the resulting records to the Writers
(in parallel).
"""
############################
def __init__(self,
readers=[],
transforms=[],
writers=[],
stderr_writers=[],
host_id='',
interval=0,
name=None,
check_format=False):
"""listener = Listener(readers, transforms=[], writers=[],
interval=0, check_format=False)
readers A single Reader or a list of Readers.
transforms A single Transform or a list of zero or more Transforms
writers A single Writer or a list of zero or more Writers
stderr_writers A single Writer or a list of zero or more Writers to which
the logger's stderr should be written.
interval How long to sleep before reading sequential records
name Optional human-readable short name to be used in displays
check_format If True, attempt to check that Reader/Transform/Writer
formats are compatible, and throw a ValueError if they
are not. If check_format is False (the default) the
output_format() of the whole reader will be
formats.Unknown.
Sample use:
listener = Listener(readers=[NetworkReader(':6221'),
NetworkReader(':6223')],
transforms=[TimestampTransform()],
writers=[TextFileWriter('/logs/network_recs'),
TextFileWriter(None)],
interval=0.2)
listener.run()
Calling listener.quit() from another thread will cause the run() loop
to exit.
"""
logging.info('Instantiating %s logger', name or 'unnamed')
###########
# Create readers, writers, etc.
self.reader = ComposedReader(readers=readers,
check_format=check_format)
self.writer = ComposedWriter(transforms=transforms,
writers=writers,
check_format=check_format)
self.interval = interval
self.name = name or 'Unnamed listener'
self.last_read = 0
self.quit_signalled = False
############################
def quit(self):
"""
Signal 'quit' to all the readers.
"""
self.quit_signalled = True
logging.info('Shutting down %s', self.name)
############################
def run(self):
"""
Read/transform/write until either quit() is called in a separate
thread, or ComposedReader returns None, indicating that all its
component readers have returned EOF.
"""
logging.info('Running %s', self.name)
if not self.reader and not self.writer:
logging.info('No readers or writers defined - exiting.')
return
record = ''
try:
while not self.quit_signalled and record is not None:
record = self.reader.read()
self.last_read = time.time()
logging.debug('ComposedReader read: "%s"', record)
if record:
self.writer.write(record)
if self.interval:
time_to_sleep = self.interval - (time.time() -
self.last_read)
time.sleep(max(time_to_sleep, 0))
# Exit in an orderly fashion if someone hits Ctl-C
except KeyboardInterrupt:
logging.info('Listener %s received KeyboardInterrupt - exiting.',
self.name or '')
except Exception as e:
logging.info('Listener %s received exception: %s', self.name,
traceback.format_exc())
raise e
disk_cache=args.disk_cache)
# Only create reader(s) if they've given us a network to read from;
# otherwise, count on data coming from websocket publish
# connections.
if args.udp:
readers = []
# Readers may either be just a port (to listen for broadcast) or
# a multicast_group:port to listen for multicast.
for udp_spec in args.udp.split(','):
group_port = udp_spec.split(':')
port = int(group_port[-1])
multicast_group = group_port[-2] if len(group_port) == 2 else ''
readers.append(UDPReader(port=port, source=multicast_group))
transform = FromJSONTransform()
reader = ComposedReader(readers=readers, transforms=[transform])
# Loop, reading data and writing it to the cache
try:
while True:
if args.udp:
record = reader.read()
server.cache_record(record)
else:
time.sleep(args.interval)
except KeyboardInterrupt:
logging.warning('Received KeyboardInterrupt - shutting down')
if args.disk_cache:
logging.warning(
'Will try to save to disk cache prior to shutdown...')