def test_collect_and_filter_events(m_save, m_send):
from time import sleep
from threading import Thread
stored_events = []
def save_event(event):
stored_events.append(event)
send_messages = []
def send_ws_message(message):
send_messages.append(message)
m_save.side_effect = save_event
m_send.side_effect = send_ws_message
ser = EventSerializer()
store = EventStore()
coll = Collector(store=store, hostname="127.0.0.1", port=1125)
def do_run():
coll.run()
t = Thread(target=do_run)
t.start()
sleep(_WAIT_TIME)
assert coll.store_loop is not None
assert coll.server_loop is not None
# Register filter
coll.server_loop.call_soon_threadsafe(coll._add_live_filter, '/live',
'{"tags": ["3"]}', WebSocket(), None)
coll.server_loop.call_soon_threadsafe(
coll._on_event, '/event',
ser.serialize(
Event(id='001',
timestamp=10,
tags=['1', '2'],
source='src1',
content='event 1')), None, None)
coll.server_loop.call_soon_threadsafe(
coll._on_event, '/event',
ser.serialize(
Event(id='002',
timestamp=20,
tags=['1', '2', '3'],
source='src2',
content='event 2')), None, None)
sleep(_WAIT_TIME)
coll.stop()
print('waiting for threads to complete...')
t.join()
assert len(stored_events) == 2
assert len(send_messages) == 1
def __init__(self, loop, host, port, secure=False, path=None, recv=None):
self.loop = loop
self.host = host
self.port = port
self.secure = secure
self.path = path
self.recv_handler = recv
self.serializer = EventSerializer()
self.websocket = None
self._is_open = False
self._close_handlers = []
def __init__(self, store, hostname='0.0.0.0', port=4300, persistent=True):
self.hostname = hostname
self.port = port
self.server = None
self.store = store
self.server_thread = None
self.store_thread = None
self.store_loop = None
self.server_loop = None
self.parser = EventParser()
self.serializer = EventSerializer()
self.live = Live(self.serializer)
self.persistent = persistent
def __init__(self, root_dir, flush_interval=1000):
self.root_dir = root_dir
self.data_file_interval = 60 # 60 seconds
self.serializer = EventSerializer()
self.index = FileIndex(root_dir)
self.open_files = {}
self.write_lock = RLock()
self.flush_interval = flush_interval # <=0 immediate, otherwise will flush periodically
self.timer = None
if flush_interval > 0:
self.timer = PeriodicTimer(flush_interval / 1000,
self._flush_open_files)
self.timer.start()
log.info('Flushing buffers every %fms', (flush_interval / 1000))
def test_server_on_action(m_send, m_recv):
loop = asyncio.new_event_loop()
ser = Server(loop=loop, host='localhost', port=11226)
serialized_event = EventSerializer().serialize(
Event(id='00001',
timestamp=10,
source='src-1',
tags=['t-1', 't-2'],
content='content-1'))
state = {}
ws = WebSocket()
async def fake_recv():
if state.get('event_yield'):
print('loop stop')
loop.stop()
await asyncio.sleep(10)
else:
state['event_yield'] = True
return serialized_event
m_recv.side_effect = fake_recv
async def fake_send(msg):
pass
m_send.side_effect = fake_send
def action_handler(path, message, websocket, resp):
assert path == '/test-path'
assert message == serialized_event
assert websocket == ws
assert resp is ''
return 'RESP'
on_action_mock = mock.MagicMock()
on_action_mock.side_effect = action_handler
ser.on_action('/test-path', on_action_mock)
ser.start()
assert ser.host == 'localhost'
assert ser.port == 11226
assert ser._started == True
asyncio.ensure_future(ser._on_client_connection(ws, '/test-path'),
loop=loop)
loop.run_forever()
for t in asyncio.Task.all_tasks(loop):
t.cancel() # cancel the asyncio.wait
print('[CANCEL]', t)
loop.close()
assert m_recv.call_count == 2 # once to get the message; second time to end the loop
assert on_action_mock.call_count == 1
assert m_send.call_count == 1
m_send.assert_called_once_with('RESP')
def test_live_pipeline(m_serialize, m_send):
"""Test event matching in the live pipeline.
"""
loop = asyncio.get_event_loop()
filtered = []
async def mock_send(data):
filtered.append(data)
def mock_serialize(event):
return "event:%s" % event.id
m_send.side_effect = mock_send
m_serialize.side_effect = mock_serialize
live = Live(serializer=EventSerializer())
live.add_filter(LiveFilter(ws=WebSocket(), criteria={'id': '1000'}))
live.add_filter(LiveFilter(ws=WebSocket(), criteria={'tags': ['a']}))
for event in [
Event(id='1000', source='s1', tags=['a', 'b', 'c']),
Event(id='2000', source='s2'),
Event(id='3000', source='s1', tags=['a', 'b']),
Event(id='4000', source='s3')
]:
loop.create_task(live.pipe(event))
loop.call_soon(loop.stop)
loop.run_forever()
assert len(filtered) == 3
def test_serialize_event(self):
ev_str = EventSerializer().serialize(
Event(id='id1',
source='env1',
tags=['a', 'b'],
content='TEST EVENT'))
expected = [
'event: 68 58 10', 'id:id1', 'timestamp:', 'source:env1',
'tags:a,b', 'TEST EVENT', ''
]
i = 0
for line in ev_str.decode('utf-8').split('\n'):
if i == 2:
assert line.startswith('timestamp:')
else:
assert line == expected[i]
i += 1
def test_collect_events(m_save):
from time import sleep
from threading import Thread
stored_events = []
def save_event(event):
stored_events.append(event)
m_save.side_effect = save_event
ser = EventSerializer()
store = EventStore()
coll = Collector(store=store, hostname="127.0.0.1", port=1124)
def do_run():
coll.run()
t = Thread(target=do_run)
t.start()
sleep(_WAIT_TIME)
assert coll.store_loop is not None
assert coll.server_loop is not None
coll.server_loop.call_soon_threadsafe(
coll._on_event, '/event',
ser.serialize(
Event(id='001',
timestamp=10,
tags=['1', '2'],
source='src1',
content='event 1')), None, None)
coll.server_loop.call_soon_threadsafe(
coll._on_event, '/event',
ser.serialize(
Event(id='002',
timestamp=20,
tags=['1', '2', '3'],
source='src2',
content='event 2')), None, None)
sleep(_WAIT_TIME)
coll.stop()
print('waiting for threads to complete...')
t.join()
assert len(stored_events) == 2
def test_event_printer(m_stdout):
from theia.model import Event, EventParser, EventSerializer
event = Event(id='id-0',
timestamp=1529233605,
source='/source/1',
tags=['a', 'b'],
content='event 1')
parser = EventParser('UTF-8')
print_event = event_printer('{id}|{timestamp}|{source}|{tags}|{content}',
'%Y-%m-%d', parser)
assert print_event is not None
print_event(EventSerializer().serialize(event))
assert m_stdout.getvalue() == 'id-0|2018-06-17|/source/1|a,b|event 1\n'
def simulate_events(args):
"""Connects and generates random events.
The events are generated according to the given arguments.
:param argparse.Namespace args: the parsed arguments passed to the program.
"""
loop = asyncio.get_event_loop()
client = Client(host=args.host, port=args.port, path='/event', loop=loop)
client.connect()
ser = EventSerializer()
tags = [args.tags] if isinstance(args.tags, str) else args.tags
class _SenderThread(Thread):
"""Sender Thread. Runs continuously.
"""
def __init__(self):
super(_SenderThread, self).__init__()
self.is_running = False
def run(self):
"""Generate and send events continuously.
"""
self.is_running = True
while self.is_running:
_send_event(args, tags, client, ser)
time.sleep(args.delay)
sender_thread = _SenderThread()
sender_thread.is_running = True
sender_thread.start()
loop.run_forever()
class NaiveEventStore(EventStore):
"""A naive implementation of the :class:`theia.storeapi.EventStore` that keeps the event data is a plain text files.
The events are kept in plain text files, serialized by default in UTF-8. The files are human readable and the format
is designed to be (relatively) easily processed by other tools as well (such as ``grep``). Each data file contains
events that happened within one minute (60000ms). The names of the data files also reflect the time span interval,
so for example a file with name *1528631988-1528632048* contains only events that happened at or after
``1528631988``, but before ``1528632048``.
The store by default uses in-memory buffers to write new events, and flushes the buffer periodically. By default
the flushing occurs roughly every second (1000ms, see the parameter ``flush_interval``). This increases the
performance of the store, but if outage occurs within this interval, the data in the in-memory buffers will be lost.
The store can be configured to flush the events immediately on disk (by passing ``0`` for ``flush_interval``), but
this decreases the performance of the store significantly.
The instances of this class are thread-safe and can be shared between threads.
:param root_dir: ``str``, the root directory where to store the events data files.
:param flush_interval: ``int``, flush interval for the data files buffers in milliseconds. The event data files will
be flushed and persisted on disk every ``flush_interval`` milliseconds. The default value is 1000ms. To flush
immediately (no buffering), set this value equal or less than ``0``.
"""
def __init__(self, root_dir, flush_interval=1000):
self.root_dir = root_dir
self.data_file_interval = 60 # 60 seconds
self.serializer = EventSerializer()
self.index = FileIndex(root_dir)
self.open_files = {}
self.write_lock = RLock()
self.flush_interval = flush_interval # <=0 immediate, otherwise will flush periodically
self.timer = None
if flush_interval > 0:
self.timer = PeriodicTimer(flush_interval / 1000,
self._flush_open_files)
self.timer.start()
log.info('Flushing buffers every %fms', (flush_interval / 1000))
def _get_event_file(self, ts_from):
data_file = self.index.find_event_file(ts_from)
if not data_file:
try:
self.write_lock.acquire()
data_file = self._get_new_data_file(ts_from)
self.index.add_file(basename(data_file.path))
finally:
self.write_lock.release()
return data_file
def _open_file(self, data_file):
return MemoryFile(name=basename(data_file.path),
path=dirname(data_file.path))
def _flush_open_files(self):
for file_name, open_file in self.open_files.items():
try:
open_file.flush()
except Exception as e:
log.error('Error while flushing %s. Error: %s', file_name, e)
def _get_new_data_file(self, ts_from):
ts_end = ts_from + self.data_file_interval
return DataFile(join_paths(self.root_dir, '%d-%d' % (ts_from, ts_end)),
ts_from, ts_end)
def save(self, event):
# lookup file/create file
# lock it
# save serialized event
# unlock
data_file = self._get_event_file(event.timestamp)
try:
self.write_lock.acquire()
if not self.open_files.get(data_file.path):
self.open_files[data_file.path] = self._open_file(data_file)
mem_file = self.open_files[data_file.path]
mem_file.write(self.serializer.serialize(event))
mem_file.write('\n'.encode(self.serializer.encoding))
if self.flush_interval <= 0:
mem_file.flush()
finally:
self.write_lock.release()
def search(self,
ts_start,
ts_end=None,
flags=None,
match=None,
order='asc'):
if order not in ['asc', 'desc']:
raise EventStoreException('Order must be one of "asc" or "desc".')
data_files = self.index.find(ts_start, ts_end)
match_fn = self._match_forward if order == 'asc' else self._match_reverse
if data_files:
for data_file in data_files:
for event in match_fn(data_file, ts_start, ts_end, flags,
match):
yield event
def close(self):
self._flush_open_files()
if self.timer:
self.timer.cancel()
self.timer.join()
log.info('Naive Store stopped')
def _search_data_file(self, data_file, ts_start, ts_end, flags, match,
reverse):
if reverse:
yield from self._match_reverse(data_file, ts_start, ts_end, flags,
match)
else:
yield from self._match_forward(data_file, ts_start, ts_end, flags,
match)
def _match_forward(self, data_file, ts_start, ts_end, flags, match):
with self._seq_event_parser(data_file) as sqp:
for event in sqp.events():
if event.timestamp >= ts_start and (ts_end is None or
event.timestamp <= ts_end):
if self._matches(event, flags, match):
yield event
def _match_reverse(self, data_file, ts_start, ts_end, flags, match):
raise Exception('Reverse order (desc) not supported right now.')
def _seq_event_parser(self, data_file):
return SequentialEventReader(open(data_file.path, 'rb'), EventParser())
def _matches(self, event, flags, match):
if flags is not None and event.tags:
for flag in flags:
if not flag in event.tags:
return False
if match and event.content and match:
if not re.search(match, event.content):
return False
return True
def get(self, event_id):
""":class:`NaiveEventStore` does not support indexing, so search by ``id`` is also not supported.
"""
log.warning(
'Get event by id [%s], but operation "get" is not supported in NaiveEventStore.',
event_id)
raise Exception('Get event not supported.')
def delete(self, event_id):
""":class:`NaiveEventStore` does not support indexing, so a delete by ``id`` is not supported.
"""
log.warning(
'Delete event by id [%s], but operation "delete" is not supported in NaiveEventStore.',
event_id)
raise Exception('Delete event not supported.')
class Collector:
"""Collector server.
Collects the events, passes them down the live pipe filters and stores them
in the event store.
:param store: :class:`theia.storeapi.Store`, store instance
:param hostame: ``str``, server hostname. Default is '0.0.0.0'.
:param port: ``int``, server port. Default is 4300.
"""
# pylint: disable=too-many-instance-attributes
def __init__(self, store, hostname='0.0.0.0', port=4300, persistent=True):
self.hostname = hostname
self.port = port
self.server = None
self.store = store
self.server_thread = None
self.store_thread = None
self.store_loop = None
self.server_loop = None
self.parser = EventParser()
self.serializer = EventSerializer()
self.live = Live(self.serializer)
self.persistent = persistent
def run(self):
"""Run the collector server.
This operation is blocking.
"""
if self.persistent:
self._setup_store()
self._setup_server()
if self.persistent:
self.store_thread.join()
else:
log.info(
'Collector will not persist the events. To persist the events please run it in persistent mode.'
)
self.server_thread.join()
def stop(self):
"""Stop the collector server.
This operation is non blocking.
"""
self.server.stop()
try:
if self.persistent:
self.store_loop.call_soon_threadsafe(self.store_loop.stop)
finally:
self.server_loop.call_soon_threadsafe(self.server_loop.stop)
if self.store:
self.store.close()
def _setup_store(self):
def run_store_thread():
"""Runs the store loop in a separate thread.
"""
loop = asyncio.new_event_loop()
self.store_loop = loop
loop.run_forever()
loop.close()
log.info('store is shut down.')
self.store_thread = Thread(target=run_store_thread)
self.store_thread.start()
def _setup_server(self):
def run_in_server_thread():
"""Runs the server loop in a separate thread.
"""
loop = asyncio.new_event_loop()
self.server_loop = loop
self.server = Server(loop=loop, host=self.hostname, port=self.port)
self.server.on_action('/event', self._on_event)
self.server.on_action('/live', self._add_live_filter)
self.server.on_action('/find', self._find_event)
self.server.start()
loop.run_forever()
loop.close()
log.info('server is shut down.')
self.server_thread = Thread(target=run_in_server_thread)
self.server_thread.start()
def _on_event(self, path, message, websocket, resp):
try:
self.store_loop.call_soon_threadsafe(self._store_event, message)
except Exception as e:
log.exception(e)
def _store_event(self, message):
event = self.parser.parse_event(BytesIO(message))
if self.persistent:
self.store.save(event)
try:
asyncio.run_coroutine_threadsafe(self.live.pipe(event),
self.server_loop)
except Exception as e:
log.error('Error in pipe: %s (event: %s)', e, event)
def _add_live_filter(self, path, message, websocket, resp):
criteria = json.loads(message)
live_filter = LiveFilter(websocket, criteria)
self.live.add_filter(live_filter)
return 'ok'
def _find_event(self, path, message, websocket, resp):
if not self.persistent:
return '{"error": "Action not available in non-persistent mode."}'
criteria = json.loads(message)
ts_from = criteria.get('start')
ts_to = criteria.get('end')
flags = criteria.get('tags')
content = criteria.get('content')
order = criteria.get('order') or 'asc'
if not ts_from:
raise Exception('Missing start timestamp')
asyncio.run_coroutine_threadsafe(
self._find_event_results(start=ts_from,
end=ts_to,
flags=flags,
match=content,
order=order,
websocket=websocket), self.store_loop)
return 'ok'
async def _find_event_results(self, start, end, flags, match, order,
websocket):
for event in self.store.search(ts_start=start,
ts_end=end,
flags=flags,
match=match,
order=order):
await self._send_result(event, websocket)
await asyncio.sleep(0, loop=self.store_loop) # let other tasks run
async def _send_result(self, event, websocket):
ser = self.serializer.serialize(event)
result = asyncio.run_coroutine_threadsafe(websocket.send(ser),
self.server_loop)
result.result()
class Client:
"""Client represents a client connection to a theia server.
:param loop: :mod:`asyncio` EventLoop to use for this client.
:param host: ``str``, theia server hostname.
:param port: ``int``, theia server port.
:param secure: ``bool``, is the connection secure.
:param path: ``str``, the request path - for example: ``"/live"``, ``"/events"`` etc.
:param recv: ``function``, receive handler. Called when a message is received from the
server. The handler has the following signature:
.. code-block:: python
def handler(message):
pass
where:
* ``message`` is the message received from the theia server.
"""
def __init__(self, loop, host, port, secure=False, path=None, recv=None):
self.loop = loop
self.host = host
self.port = port
self.secure = secure
self.path = path
self.recv_handler = recv
self.serializer = EventSerializer()
self.websocket = None
self._is_open = False
self._close_handlers = []
async def _open_websocket(self):
websocket = await websockets.connect(self._get_ws_url(), loop=self.loop)
self.websocket = websocket
self._is_open = True
asyncio.ensure_future(self._recv(), loop=self.loop)
log.debug('[%s:%d]: connected', self.host, self.port)
def connect(self):
"""Connect to the remote server.
"""
self.loop.run_until_complete(self._open_websocket())
def close(self, reason=None):
"""Close the connection to the remote server.
:param reason: ``str``, the reason for disconnecting. If not given, a default ``"normal close"`` is
sent to the server.
"""
reason = reason or 'normal close'
self._is_open = False
self.websocket.close(code=1000, reason=reason)
log.debug('[%s:%d]: explicitly closed. Reason=%s', self.host, self.port, reason)
def _get_ws_url(self):
url = 'wss://' if self.secure else 'ws://'
url += self.host
if self.port:
url += ':' + str(self.port)
if self.path:
if self.path.startswith('/'):
url += self.path
else:
url += '/' + self.path
return url
def send(self, message):
"""Send a ``str`` message to the remote server.
:param message: ``str``, the message to be sent to the remote server.
:returns: the :class:`asyncio.Handle` to the scheduled task for sending the
actual data.
"""
return self.loop.call_soon_threadsafe(self._call_send, message)
def _call_send(self, message):
asyncio.ensure_future(self.websocket.send(message), loop=self.loop)
def send_event(self, event):
"""Send an event to the remote server.
Serializes, then sends the serialized content to the remote server.
:param event: :class:`theia.model.Event`, the event to be send.
:returns: the :class:`asyncio.Handle` to the scheduled task for sending the
actual data.
"""
message = self.serializer.serialize(event)
return self.send(message)
async def _recv(self):
while self._is_open:
try:
message = await self.websocket.recv()
await self._process_message(message)
except websockets.ConnectionClosed as wse:
self._closed(wse.code, wse.reason)
log.debug('[%s:%d] connection closed', self.host, self.port)
# pylint: disable=broad-except
# General case
except Exception as e:
log.exception(e)
self._closed(1006, reason=str(e))
async def _process_message(self, message):
if self.recv_handler:
self.recv_handler(message)
def on_close(self, handler):
"""Add close handler.
The handles is called when the client connection is closed either by the client
or by the server.
:param handler: ``function``, the handler callback. The callback prototype
looks like so:
.. code-block:: python
def callback(websocket, code, reason):
pass
where:
* ``websocket`` :class:`websockets.WebSocketClientProtocol` is the underlying
websocket.
* ``code`` ``int`` is the code received when the connection was closed. Check
out the `WebSocket specification`_ for the list of codes and their meaning.
* ``reason`` ``str`` is the reason for closing the connection.
.. _WebSocket specification: https://tools.ietf.org/html/rfc6455#section-7.4
"""
self._close_handlers.append(handler)
def _closed(self, code=1000, reason=None):
self._is_open = False
for hnd in self._close_handlers:
try:
hnd(self.websocket, code, reason)
except Exception as e:
log.debug(e)
def is_open(self):
"""Check if the client connection is open.
:returns: ``True`` if the client connection is open, otherwise ``False``.
"""
return self._is_open
def test_server_close_handler(m_close, m_send, m_recv):
from threading import Thread
from time import sleep
loop = asyncio.new_event_loop()
ser = Server(loop=loop, host='localhost', port=11227)
ser._stop_timeout = 0.1
serialized_event = EventSerializer().serialize(
Event(id='00001',
timestamp=10,
source='src-1',
tags=['t-1', 't-2'],
content='content-1'))
state = {}
websocket = WebSocket()
def ws_close_handler(ws, path):
assert ws == webocket
assert path == 'test-path'
mock_handler = mock.MagicMock()
mock_handler.side_effect = ws_close_handler
async def fake_recv():
# at this point, the client websocket is registered.
success = ser.on_websocket_close(websocket, mock_handler)
assert success
loop.call_soon(ser.stop)
await asyncio.sleep(0.4)
loop.call_soon(loop.stop)
await asyncio.sleep(10)
m_recv.side_effect = fake_recv
async def fake_send(msg):
pass
m_send.side_effect = fake_send
ser.start()
assert ser.host == 'localhost'
assert ser.port == 11227
assert ser._started == True
asyncio.ensure_future(ser._on_client_connection(websocket, 'test-path'),
loop=loop)
loop.run_forever()
print(' *** loop done ***')
for t in asyncio.Task.all_tasks(loop):
t.cancel() # cancel the asyncio.wait
print('[CANCEL]', t)
loop.close()
assert m_recv.call_count == 1
assert mock_handler.call_count == 1
assert m_close.call_count == 1
def test_naive_store_search():
known_events = [{"name": "10-70", "events": [Event(id="event-1", timestamp=10, source="/src/1", tags=["a"], content="event-1, data file 1"),
Event(id="event-2", timestamp=15, source="/src/2", tags=["b"], content="event-2, data file 1"),
Event(id="event-3", timestamp=30, source="/src/1", tags=["a", "b"], content="event-3, data file 1"),
Event(id="event-4", timestamp=67, source="/src/3", tags=["c" ], content="event-4, data file 1")]},
{"name": "71-131", "events": [Event(id="event-5", timestamp=75, source="/src/1", tags=["d"], content="event-5, data file 2"),
Event(id="event-6", timestamp=100, source="/src/4", tags=["e" ], content="event-6, data file 2")]},
{"name": "200-260", "events": [Event(id="event-7", timestamp=200, source="/src/5", tags=["f"], content="event-7, data file 3"),
Event(id="event-8", timestamp=210, source="/src/2", tags=["g"], content="event-8, data file 3"),
Event(id="event-9", timestamp=220, source="/src/1", tags=["h", "f"], content="event-9, data file 3"),
Event(id="event-10", timestamp=250, source="/src/6", tags=["i" ], content="event-10, data file 3")]}]
serializer = EventSerializer()
with tempfile.TemporaryDirectory() as tmpdir:
# create data files with events
for df_spec in known_events:
with open(os.path.join(tmpdir, df_spec["name"]), "wb") as df:
for event in df_spec["events"]:
df.write(serializer.serialize(event))
df.write("\n".encode("utf-8")) # the extra newline to separate events in the data file
ns = NaiveEventStore(root_dir=tmpdir)
try:
# lookup all
results = []
for event in ns.search(ts_start=10):
results.append(event)
assert len(results) == 10
# lookup in an interval
results = []
for event in ns.search(ts_start=30, ts_end=80):
results.append(event)
assert len(results) == 3
assert [e.id for e in results] == ['event-3', 'event-4', 'event-5']
# lookup by tags
results = []
for event in ns.search(ts_start=5, flags=["a"]):
results.append(event)
assert len(results) == 2
assert [e.id for e in results] == ['event-1', 'event-3']
# lookup by content
results = []
for event in ns.search(ts_start=5, match='data file 3'):
results.append(event)
assert len(results) == 4
assert [e.id for e in results] == ['event-7', 'event-8', 'event-9', 'event-10']
# lookup by content - regex through multiple files
results = []
for event in ns.search(ts_start=5, match='event-(4|5|7|10)'):
results.append(event)
assert len(results) == 4
assert [e.id for e in results] == ['event-4', 'event-5', 'event-7', 'event-10']
finally:
if ns:
ns.close()