def __init__(self, store, hostname='0.0.0.0', port=4300, persistent=True):
self.hostname = hostname
self.port = port
self.server = None
self.store = store
self.server_thread = None
self.store_thread = None
self.store_loop = None
self.server_loop = None
self.parser = EventParser()
self.serializer = EventSerializer()
self.live = Live(self.serializer)
self.persistent = persistent
def test_parse_event(self):
event_str = '\n'.join([
'event: 68 58 10', 'id:id1', 'timestamp: 1491580705.9789374',
'source:env1', 'tags:a,b', 'TEST EVENT'
])
parser = EventParser()
event = parser.parse_event(BytesIO(event_str.encode('utf-8')))
assert event
assert event.id == 'id1'
assert event.source == 'env1'
assert event.timestamp == 1491580705.9789374
assert event.tags == ['a', 'b']
assert event.content == 'TEST EVENT'
def run_query(args):
"""Configures and runs a :class:`theia.query.Query`.
:param argparse.Namespace args: parsed command-line arguments.
"""
loop = asyncio.get_event_loop()
query = Query(host=args.server_host, port=args.port, loop=loop)
criteria_filter = _to_criteria(args)
result = None
parser = EventParser('UTF-8')
if args.live:
result = query.live(criteria_filter, event_printer(args.o_format,
args.o_ts_format,
parser))
else:
result = query.find(criteria_filter, event_printer(args.o_format,
args.o_ts_format,
parser))
def clean_stop(*arg):
"""Perform a clean stop - wait a bit for the tasks on the loop to finish.
"""
# delay the stop a bit, give it chance to actually close the loop
loop.call_later(args.close_timeout/1000, loop.stop)
result.when_closed(clean_stop)
loop.add_signal_handler(signal.SIGHUP, loop.stop)
loop.add_signal_handler(signal.SIGINT, loop.stop)
loop.add_signal_handler(signal.SIGTERM, loop.stop)
loop.run_forever()
loop.close()
def test_sequential_event_reader_curr_event(m_parse_event):
def fake_parse_event(stream, skip_content):
assert skip_content is False
return Event(id='event_zero', source='/source/_zero_event')
m_parse_event.side_effect = fake_parse_event
reader = SequentialEventReader(stream=BytesIO(), event_parser=EventParser())
ev = reader.curr_event()
assert ev is not None
assert m_parse_event.call_count is 1
def test_event_printer(m_stdout):
from theia.model import Event, EventParser, EventSerializer
event = Event(id='id-0',
timestamp=1529233605,
source='/source/1',
tags=['a', 'b'],
content='event 1')
parser = EventParser('UTF-8')
print_event = event_printer('{id}|{timestamp}|{source}|{tags}|{content}',
'%Y-%m-%d', parser)
assert print_event is not None
print_event(EventSerializer().serialize(event))
assert m_stdout.getvalue() == 'id-0|2018-06-17|/source/1|a,b|event 1\n'
def test_sequential_event_reader_as_context_manager(m_parse_event):
class _FakeBytesIO:
def close(self):
pass
def fake_parse_event(stream, skip_content):
assert skip_content is False
return Event(id='event_zero', source='/source/_zero_event')
m_parse_event.side_effect = fake_parse_event
stream = _FakeBytesIO()
stream.close = mock.MagicMock()
with SequentialEventReader(stream=stream, event_parser=EventParser()) as reader:
ev = reader.curr_event()
assert ev is not None
assert m_parse_event.call_count is 1
assert stream.close.call_count is 1
def test_sequential_event_reader_events(m_parse_event):
state = {}
total_events = 3
def fake_parse_event(stream, skip_content):
state['faked'] = state.get('faked', 0)
if state['faked'] == total_events:
raise EOFException('break')
state['faked'] += 1
return Event(id='event%d' % state['faked'], source='/source/%d' % state['faked'])
m_parse_event.side_effect = fake_parse_event
reader = SequentialEventReader(stream=BytesIO(), event_parser=EventParser())
events = []
for ev in reader.events():
events.append(ev)
assert len(events) == 3
assert m_parse_event.call_count == 4
def _seq_event_parser(self, data_file):
return SequentialEventReader(open(data_file.path, 'rb'), EventParser())
class Collector:
"""Collector server.
Collects the events, passes them down the live pipe filters and stores them
in the event store.
:param store: :class:`theia.storeapi.Store`, store instance
:param hostame: ``str``, server hostname. Default is '0.0.0.0'.
:param port: ``int``, server port. Default is 4300.
"""
# pylint: disable=too-many-instance-attributes
def __init__(self, store, hostname='0.0.0.0', port=4300, persistent=True):
self.hostname = hostname
self.port = port
self.server = None
self.store = store
self.server_thread = None
self.store_thread = None
self.store_loop = None
self.server_loop = None
self.parser = EventParser()
self.serializer = EventSerializer()
self.live = Live(self.serializer)
self.persistent = persistent
def run(self):
"""Run the collector server.
This operation is blocking.
"""
if self.persistent:
self._setup_store()
self._setup_server()
if self.persistent:
self.store_thread.join()
else:
log.info(
'Collector will not persist the events. To persist the events please run it in persistent mode.'
)
self.server_thread.join()
def stop(self):
"""Stop the collector server.
This operation is non blocking.
"""
self.server.stop()
try:
if self.persistent:
self.store_loop.call_soon_threadsafe(self.store_loop.stop)
finally:
self.server_loop.call_soon_threadsafe(self.server_loop.stop)
if self.store:
self.store.close()
def _setup_store(self):
def run_store_thread():
"""Runs the store loop in a separate thread.
"""
loop = asyncio.new_event_loop()
self.store_loop = loop
loop.run_forever()
loop.close()
log.info('store is shut down.')
self.store_thread = Thread(target=run_store_thread)
self.store_thread.start()
def _setup_server(self):
def run_in_server_thread():
"""Runs the server loop in a separate thread.
"""
loop = asyncio.new_event_loop()
self.server_loop = loop
self.server = Server(loop=loop, host=self.hostname, port=self.port)
self.server.on_action('/event', self._on_event)
self.server.on_action('/live', self._add_live_filter)
self.server.on_action('/find', self._find_event)
self.server.start()
loop.run_forever()
loop.close()
log.info('server is shut down.')
self.server_thread = Thread(target=run_in_server_thread)
self.server_thread.start()
def _on_event(self, path, message, websocket, resp):
try:
self.store_loop.call_soon_threadsafe(self._store_event, message)
except Exception as e:
log.exception(e)
def _store_event(self, message):
event = self.parser.parse_event(BytesIO(message))
if self.persistent:
self.store.save(event)
try:
asyncio.run_coroutine_threadsafe(self.live.pipe(event),
self.server_loop)
except Exception as e:
log.error('Error in pipe: %s (event: %s)', e, event)
def _add_live_filter(self, path, message, websocket, resp):
criteria = json.loads(message)
live_filter = LiveFilter(websocket, criteria)
self.live.add_filter(live_filter)
return 'ok'
def _find_event(self, path, message, websocket, resp):
if not self.persistent:
return '{"error": "Action not available in non-persistent mode."}'
criteria = json.loads(message)
ts_from = criteria.get('start')
ts_to = criteria.get('end')
flags = criteria.get('tags')
content = criteria.get('content')
order = criteria.get('order') or 'asc'
if not ts_from:
raise Exception('Missing start timestamp')
asyncio.run_coroutine_threadsafe(
self._find_event_results(start=ts_from,
end=ts_to,
flags=flags,
match=content,
order=order,
websocket=websocket), self.store_loop)
return 'ok'
async def _find_event_results(self, start, end, flags, match, order,
websocket):
for event in self.store.search(ts_start=start,
ts_end=end,
flags=flags,
match=match,
order=order):
await self._send_result(event, websocket)
await asyncio.sleep(0, loop=self.store_loop) # let other tasks run
async def _send_result(self, event, websocket):
ser = self.serializer.serialize(event)
result = asyncio.run_coroutine_threadsafe(websocket.send(ser),
self.server_loop)
result.result()