def main():
"""Test script"""
import numpy as np
from bcipy.acquisition.record import Record
import timeit
n_rows = 1000
channel_count = 25
channels = ["ch" + str(c) for c in range(channel_count)]
pid1 = start(channels, 'buffer1.db')
pid2 = start(channels, 'buffer2.db')
starttime = timeit.default_timer()
for i in range(n_rows):
data = [np.random.uniform(-1000, 1000) for _ in range(channel_count)]
if i % 2 == 0:
append(pid1, Record(data, i, None))
else:
append(pid2, Record(data, i, None))
endtime = timeit.default_timer()
totaltime = endtime - starttime
print("Records inserted in buffer 1: {}".format(count(pid1)))
print("Records inserted in buffer 2: {}".format(count(pid2)))
print("Total insert time: " + str(totaltime))
query_n = 5
data = get_data(pid1, 0, query_n)
print("Sample records from buffer 1 (query < {}): {}".format(
query_n, data))
stop(pid1)
stop(pid2)
def test_query_data(self):
"""Test querying for data."""
n_records = 20
channels = ["ch1", "ch2", "TRG"]
trg_index = -1
channel_count = len(channels)
buf = Buffer(channels=channels)
for i, data in enumerate(mock_data(n_records, channel_count)):
data[trg_index] = 1.0 if i >= 10 else 0.0
timestamp = float(i)
buf.append(Record(data, timestamp, None))
rows = buf.query_data(filters=[("TRG", ">", 0)],
ordering=("timestamp", "asc"),
max_results=1)
self.assertEqual(len(rows), 1, "Should have limited to max_results.")
self.assertEqual(rows[0].data[trg_index], 1.0,
"Should get filtered data.")
self.assertEqual(rows[0].timestamp, 10.0,
"Should get first instance")
buf.cleanup()
def test_count(self):
n = 500
for i in range(n):
d = self._new_data()
buffer_server.append(self.pid,
Record(data=d, timestamp=i, rownum=None))
self.assertEqual(buffer_server.count(self.pid), n)
def test_count(self):
"""Test that the count of records is correct."""
n_records = 500
for i, data in enumerate(mock_data(n_records, self.channel_count)):
buffer_server.append(self.pid,
Record(data=data, timestamp=i, rownum=None))
self.assertEqual(buffer_server.count(self.pid), n_records)
def _convert_row(row):
"""Convert from database row to Record.
Parameters
----------
row : tuple
"""
return Record(data=list(row[2:]), timestamp=row[1], rownum=row[0])
def test_get_all_data(self):
n = 150
data = [self._new_data() for x in range(n)]
for i, d in enumerate(data):
buffer_server.append(self.pid,
Record(data=d, timestamp=i, rownum=None))
result = buffer_server.get_data(self.pid)
self.assertEqual([r.data for r in result], data, "Should return all \
data")
def test_multiple_servers(self):
"""Test multiple concurrent servers."""
pid2 = buffer_server.start(self.channels, self._next_buf_name())
n_records = 200
for count, data in enumerate(mock_data(n_records, self.channel_count)):
if count % 2 == 0:
buffer_server.append(self.pid, Record(data, count, None))
else:
buffer_server.append(pid2, Record(data, count, None))
self.assertEqual(buffer_server.count(self.pid), n_records / 2)
self.assertEqual(buffer_server.count(pid2), n_records / 2)
server1_data = buffer_server.get_data(self.pid, 0, 5)
server2_data = buffer_server.get_data(pid2, 0, 5)
self.assertNotEqual(server1_data, server2_data)
buffer_server.stop(pid2)
def test_get_all_data(self):
"""Test method to get all data from buffer."""
data = list(mock_data(n_records=150, n_cols=self.channel_count))
for record_index, record in enumerate(data):
buffer_server.append(
self.pid,
Record(data=record, timestamp=record_index, rownum=None))
result = buffer_server.get_data(self.pid)
self.assertEqual([r.data for r in result], data, "Should return all \
data")
def test_len(self):
"""Test buffer len."""
n = 1000
channel_count = 25
channels = ["ch" + str(c) for c in range(channel_count)]
b = Buffer(channels=channels)
for i, d in enumerate(_mockdata(n, channel_count)):
b.append(Record(d, float(i), None))
self.assertEqual(len(b), n)
b.cleanup()
def test_len(self):
"""Test buffer len."""
n_records = 1000
channel_count = 25
channels = ["ch" + str(c) for c in range(channel_count)]
buf = Buffer(channels=channels)
for i, data in enumerate(mock_data(n_records, channel_count)):
buf.append(Record(data, float(i), None))
self.assertEqual(len(buf), n_records)
buf.cleanup()
def test_multiple_servers(self):
pid2 = buffer_server.start(self.channels, self._next_buf_name())
n = 200
for i in range(n):
d = [
np.random.uniform(-1000, 1000)
for cc in range(self.channel_count)
]
if i % 2 == 0:
buffer_server.append(self.pid, Record(d, i, None))
else:
buffer_server.append(pid2, Record(d, i, None))
self.assertEqual(buffer_server.count(self.pid), n / 2)
self.assertEqual(buffer_server.count(pid2), n / 2)
server1_data = buffer_server.get_data(self.pid, 0, 5)
server2_data = buffer_server.get_data(pid2, 0, 5)
self.assertNotEqual(server1_data, server2_data)
buffer_server.stop(pid2)
def test_query_with_invalid_filter_field(self):
n = 20
channels = ["ch1", "ch2", "TRG"]
channel_count = len(channels)
b = Buffer(channels=channels)
for i, d in enumerate(_mockdata(n, channel_count)):
timestamp = float(i)
b.append(Record(d, timestamp, None))
with pytest.raises(Exception):
b.query_data(filters=[("ch3", ">", 0)])
b.cleanup()
def test_get_data_slice(self):
n = 150
data = [self._new_data() for x in range(n)]
for i, d in enumerate(data):
buffer_server.append(self.pid,
Record(data=d, timestamp=i, rownum=None))
start = 10
end = 20
result = buffer_server.get_data(self.pid,
start,
end,
field='timestamp')
self.assertEqual([r.data for r in result], data[start:end], "Should \
return the slice of data requested.")
def test_query_with_invalid_filter_field(self):
"""Test query with invalid filter field."""
n_records = 20
channels = ["ch1", "ch2", "TRG"]
channel_count = len(channels)
buf = Buffer(channels=channels)
for i, data in enumerate(mock_data(n_records, channel_count)):
timestamp = float(i)
buf.append(Record(data, timestamp, None))
with pytest.raises(Exception):
buf.query_data(filters=[("ch3", ">", 0)])
buf.cleanup()
def _main():
import argparse
import timeit
import numpy as np
parser = argparse.ArgumentParser()
parser.add_argument('-n', '--n_records', default=100000, type=int,
help='number of records to insert; default is 100000')
parser.add_argument('-s', '--chunk_size', default=10000, type=int,
help="default is 10000")
parser.add_argument('-c', '--channel_count', default=25, type=int,
help="default is 25")
args = parser.parse_args()
n = args.n_records
chunksize = args.chunk_size
channel_count = args.channel_count
channels = ["ch" + str(c) for c in range(channel_count)]
print("Running with %d samples of %d channels each and chunksize %d" %
(n, channel_count, chunksize))
b = Buffer(channels=channels, chunksize=chunksize)
def data(n, c):
"""Generater for mock data"""
i = 0
while i < n:
yield [np.random.uniform(-1000, 1000) for cc in range(c)]
i += 1
starttime = timeit.default_timer()
for j, d in enumerate(data(n, channel_count)):
timestamp = timeit.default_timer()
b.append(Record(d, timestamp, None))
endtime = timeit.default_timer()
totaltime = endtime - starttime
print("Total records inserted: " + str(len(b)))
print("Total time: " + str(totaltime))
print("Records per second: " + str(n / totaltime))
print("First 5 records")
print(b.query(start=0, end=6))
b.cleanup()
def test_query_data(self):
n = 150
data = [self._new_data() for x in range(n)]
last_channel = self.channels[-1]
for i, d in enumerate(data):
d[-1] = 1.0 if i >= 100 else 0.0
buffer_server.append(self.pid,
Record(data=d, timestamp=i, rownum=None))
result = buffer_server.query(self.pid,
filters=[(last_channel, ">", 0)],
ordering=("timestamp", "asc"),
max_results=1)
self.assertEqual(len(result), 1)
self.assertEqual(result[0].data[-1], 1.0)
self.assertEqual(result[0].timestamp, 100.0)
def test_get_data_slice(self):
"""Test querying for a slice of data."""
data = list(mock_data(n_records=150, n_cols=self.channel_count))
for i, record in enumerate(data):
buffer_server.append(self.pid,
Record(data=record, timestamp=i, rownum=None))
start = 10
end = 20
result = buffer_server.get_data(self.pid,
start,
end,
field='timestamp')
self.assertEqual([r.data for r in result], data[start:end], "Should \
return the slice of data requested.")
def test_query_with_invalid_order_direction(self):
n = 20
channels = ["ch1", "ch2", "TRG"]
trg_index = -1
channel_count = len(channels)
b = Buffer(channels=channels)
for i, d in enumerate(_mockdata(n, channel_count)):
d[trg_index] = 1.0 if i >= 10 else 0.0
timestamp = float(i)
b.append(Record(d, timestamp, None))
with pytest.raises(Exception):
b.query_data(ordering=("ch1", "ascending"))
b.cleanup()
def _main():
import argparse
import timeit
from bcipy.acquisition.util import mock_data
parser = argparse.ArgumentParser()
parser.add_argument('-n',
'--n_records',
default=100000,
type=int,
help='number of records to insert; default is 100000')
parser.add_argument('-s',
'--chunk_size',
default=10000,
type=int,
help="default is 10000")
parser.add_argument('-c',
'--channel_count',
default=25,
type=int,
help="default is 25")
args = parser.parse_args()
channels = ["ch" + str(c) for c in range(args.channel_count)]
print((f"Running with {args.n_records} samples of {args.channel_count} ",
f"channels each and chunksize {args.chunk_size}"))
buf = Buffer(channels=channels, chunksize=args.chunk_size)
starttime = timeit.default_timer()
for record_data in mock_data(args.n_records, args.channel_count):
timestamp = timeit.default_timer()
buf.append(Record(record_data, timestamp, None))
endtime = timeit.default_timer()
totaltime = endtime - starttime
print("Total records inserted: " + str(len(buf)))
print("Total time: " + str(totaltime))
print("Records per second: " + str(args.n_records / totaltime))
print("First 5 records")
print(buf.query(start=0, end=6))
buf.cleanup()
def test_query_with_invalid_order_direction(self):
"""Test query with invalid order direction"""
n_records = 20
channels = ["ch1", "ch2", "TRG"]
trg_index = -1
channel_count = len(channels)
buf = Buffer(channels=channels)
for i, data in enumerate(mock_data(n_records, channel_count)):
data[trg_index] = 1.0 if i >= 10 else 0.0
timestamp = float(i)
buf.append(Record(data, timestamp, None))
with pytest.raises(Exception):
buf.query_data(ordering=("ch1", "ascending"))
buf.cleanup()
def run(self):
"""Process startup. Connects to the device and start reading data.
Since this is done in a separate process from the main thread, any
errors encountered will be written to the msg_queue.
"""
try:
log.debug("Connecting to device")
self._connector.connect()
self._connector.acquisition_init()
except Exception as error:
self.msg_queue.put((MSG_ERROR, str(error)))
raise error
# Send updated device info to the main thread; this also signals that
# initialization is complete.
self.msg_queue.put((MSG_DEVICE_INFO, self._connector.device_info))
# Wait for db server start
self.msg_queue.get()
self.msg_queue = None
log.debug("Starting Acquisition read data loop")
sample = 0
data = self._connector.read_data()
# begin continuous acquisition process as long as data received
while self.running() and data:
sample += 1
if DEBUG and sample % DEBUG_FREQ == 0:
log.debug("Read sample: %s", str(sample))
buffer_server.append(self._buf,
Record(data, self._clock.getTime(), sample))
try:
# Read data again
data = self._connector.read_data()
# pylint: disable=broad-except
except Exception as error:
log.error("Error reading data from device: %s", str(error))
data = None
break
log.debug("Total samples read: %s", str(sample))
self._connector.disconnect()
def test_query_data(self):
"""Test query_data method"""
data = list(mock_data(n_records=150, n_cols=self.channel_count))
last_channel = self.channels[-1]
for record_index, record in enumerate(data):
record[-1] = 1.0 if record_index >= 100 else 0.0
buffer_server.append(
self.pid,
Record(data=record, timestamp=record_index, rownum=None))
result = buffer_server.query(self.pid,
filters=[(last_channel, ">", 0)],
ordering=("timestamp", "asc"),
max_results=1)
self.assertEqual(len(result), 1)
self.assertEqual(result[0].data[-1], 1.0)
self.assertEqual(result[0].timestamp, 100.0)
def test_query_before_flush(self):
"""If a query is made before chunksize records have been written,
the data should still be available."""
n = 1000
channel_count = 25
channels = ["ch" + str(c) for c in range(channel_count)]
b = Buffer(channels=channels, chunksize=10000)
for i, d in enumerate(_mockdata(n, channel_count)):
timestamp = float(i)
b.append(Record(d, timestamp, None))
rows = b.query(start=b.start_time, field='timestamp')
self.assertEqual(len(rows), n)
self.assertEqual(len(b.all()), n)
b.cleanup()
def test_latest(self):
"""Test query for most recent items."""
n = 1000
latest_n = 100
channel_count = 25
channels = ["ch" + str(c) for c in range(channel_count)]
b = Buffer(channels=channels)
latest = []
for i, d in enumerate(_mockdata(n, channel_count)):
timestamp = float(i)
if i >= n - latest_n:
latest.append((d, timestamp, i + 1))
b.append(Record(d, timestamp, None))
rows = b.latest(latest_n)
for j, item in enumerate(reversed(latest)):
self.assertEqual(item, rows[j])
b.cleanup()
def test_buffer(self):
"""Test Buffer functionality."""
n_records = 15000
channel_count = 25
channels = ["ch" + str(c) for c in range(channel_count)]
buf = Buffer(channels=channels, chunksize=10000)
append_timer = _Timer()
timevalues = {}
for i, data in enumerate(mock_data(n_records, channel_count)):
timestamp = float(i)
if i % 1000 == 0:
timevalues[timestamp] = data
with append_timer:
buf.append(Record(data, timestamp, None))
self.assertEqual(buf.start_time, 0.0)
starttime = 0.0
rows = buf.query(start=starttime, end=starttime +
1.0, field='timestamp')
self.assertEqual(len(rows), 1, "Results should not include end value.")
self.assertEqual(rows[0].timestamp, starttime)
self.assertEqual(rows[0].data, timevalues[starttime])
start_time = 1000.0
end_time = 2000.0
rows = buf.query(start=start_time, end=end_time, field='timestamp')
self.assertEqual(len(rows), start_time)
self.assertEqual(rows[0].data, timevalues[start_time])
rows = buf.query(start=buf.start_time, field='timestamp')
self.assertEqual(
len(rows), n_records, ("Providing only the start should return "
"the rest."))
buf.cleanup()
def run(self):
"""Continuously reads data from the source and sends it to the buffer
for processing.
"""
try:
logging.debug("Connecting to device")
self._device.connect()
self._device.acquisition_init()
except Exception as e:
self._msg_queue.put((MSG_ERROR, str(e)))
raise e
# Send updated device info to the main thread; this also signals that
# initialization is complete.
self._msg_queue.put((MSG_DEVICE_INFO, self._device.device_info))
logging.debug("Starting Acquisition read data loop")
sample = 0
data = self._device.read_data()
# begin continuous acquisition process as long as data received
while self.running() and data:
sample += 1
if DEBUG and sample % DEBUG_FREQ == 0:
logging.debug("Read sample: " + str(sample))
self._data_queue.put(Record(data, self._clock.getTime(), sample))
try:
# Read data again
data = self._device.read_data()
except Exception as e:
logging.debug("Error reading data from device: " + str(e))
data = None
break
logging.debug("Total samples read: " + str(sample))
self._device.disconnect()
