def test_query_data(self):
"""Test querying for data."""
n_records = 20
channels = ["ch1", "ch2", "TRG"]
trg_index = -1
channel_count = len(channels)
buf = Buffer(channels=channels)
for i, data in enumerate(mock_data(n_records, channel_count)):
data[trg_index] = 1.0 if i >= 10 else 0.0
timestamp = float(i)
buf.append(Record(data, timestamp, None))
rows = buf.query_data(filters=[("TRG", ">", 0)],
ordering=("timestamp", "asc"),
max_results=1)
self.assertEqual(len(rows), 1, "Should have limited to max_results.")
self.assertEqual(rows[0].data[trg_index], 1.0,
"Should get filtered data.")
self.assertEqual(rows[0].timestamp, 10.0,
"Should get first instance")
buf.cleanup()
def test_count(self):
"""Test that the count of records is correct."""
n_records = 500
for i, data in enumerate(mock_data(n_records, self.channel_count)):
buffer_server.append(self.pid,
Record(data=data, timestamp=i, rownum=None))
self.assertEqual(buffer_server.count(self.pid), n_records)
def __init__(self, *args, **kwargs):
super(TestDataAcquisitionClient, self).__init__(*args, **kwargs)
num_channels = 25
num_records = 500
self.device_spec = DeviceSpec(
name="Mock_device",
channels=['ch' + str(i) for i in range(num_channels)],
sample_rate=300.0)
self.mock_data = list(mock_data(num_records, num_channels))
def test_get_all_data(self):
"""Test method to get all data from buffer."""
data = list(mock_data(n_records=150, n_cols=self.channel_count))
for record_index, record in enumerate(data):
buffer_server.append(
self.pid,
Record(data=record, timestamp=record_index, rownum=None))
result = buffer_server.get_data(self.pid)
self.assertEqual([r.data for r in result], data, "Should return all \
data")
def test_len(self):
"""Test buffer len."""
n_records = 1000
channel_count = 25
channels = ["ch" + str(c) for c in range(channel_count)]
buf = Buffer(channels=channels)
for i, data in enumerate(mock_data(n_records, channel_count)):
buf.append(Record(data, float(i), None))
self.assertEqual(len(buf), n_records)
buf.cleanup()
def test_file_generator(self):
"""Should stream data from a file."""
row_count = 100
header = ['col1,col2,col3']
data = list(mock_data(row_count, len(header)))
rows = map(lambda x: ','.join(map(str, x)), data)
test_data = '\n'.join(header + rows)
with patch('bcipy.acquisition.datastream.generator.open',
mock_open(read_data=test_data), create=True):
gen = file_data(filename='foo', header_row=1)
generated_data = [next(gen) for _ in range(row_count)]
for i, row in enumerate(generated_data):
self.assertEqual(row, data[i])
def test_query_with_invalid_filter_field(self):
"""Test query with invalid filter field."""
n_records = 20
channels = ["ch1", "ch2", "TRG"]
channel_count = len(channels)
buf = Buffer(channels=channels)
for i, data in enumerate(mock_data(n_records, channel_count)):
timestamp = float(i)
buf.append(Record(data, timestamp, None))
with pytest.raises(Exception):
buf.query_data(filters=[("ch3", ">", 0)])
buf.cleanup()
def test_get_data_slice(self):
"""Test querying for a slice of data."""
data = list(mock_data(n_records=150, n_cols=self.channel_count))
for i, record in enumerate(data):
buffer_server.append(self.pid,
Record(data=record, timestamp=i, rownum=None))
start = 10
end = 20
result = buffer_server.get_data(self.pid,
start,
end,
field='timestamp')
self.assertEqual([r.data for r in result], data[start:end], "Should \
return the slice of data requested.")
def _main():
import argparse
import timeit
from bcipy.acquisition.util import mock_data
parser = argparse.ArgumentParser()
parser.add_argument('-n',
'--n_records',
default=100000,
type=int,
help='number of records to insert; default is 100000')
parser.add_argument('-s',
'--chunk_size',
default=10000,
type=int,
help="default is 10000")
parser.add_argument('-c',
'--channel_count',
default=25,
type=int,
help="default is 25")
args = parser.parse_args()
channels = ["ch" + str(c) for c in range(args.channel_count)]
print((f"Running with {args.n_records} samples of {args.channel_count} ",
f"channels each and chunksize {args.chunk_size}"))
buf = Buffer(channels=channels, chunksize=args.chunk_size)
starttime = timeit.default_timer()
for record_data in mock_data(args.n_records, args.channel_count):
timestamp = timeit.default_timer()
buf.append(Record(record_data, timestamp, None))
endtime = timeit.default_timer()
totaltime = endtime - starttime
print("Total records inserted: " + str(len(buf)))
print("Total time: " + str(totaltime))
print("Records per second: " + str(args.n_records / totaltime))
print("First 5 records")
print(buf.query(start=0, end=6))
buf.cleanup()
def test_query_with_invalid_order_direction(self):
"""Test query with invalid order direction"""
n_records = 20
channels = ["ch1", "ch2", "TRG"]
trg_index = -1
channel_count = len(channels)
buf = Buffer(channels=channels)
for i, data in enumerate(mock_data(n_records, channel_count)):
data[trg_index] = 1.0 if i >= 10 else 0.0
timestamp = float(i)
buf.append(Record(data, timestamp, None))
with pytest.raises(Exception):
buf.query_data(ordering=("ch1", "ascending"))
buf.cleanup()
def test_file_generator_end(self):
"""Should throw an exception when all data has been consumed"""
row_count = 10
header = ['col1,col2,col3']
data = list(mock_data(row_count, len(header)))
rows = map(lambda x: ','.join(map(str, x)), data)
test_data = '\n'.join(header + rows)
with patch('bcipy.acquisition.datastream.generator.open',
mock_open(read_data=test_data), create=True):
gen = file_data(filename='foo', header_row=1)
# exhaust the generator
for _ in range(row_count):
next(gen)
with pytest.raises(StopIteration):
data.append(next(gen))
def test_query_data(self):
"""Test query_data method"""
data = list(mock_data(n_records=150, n_cols=self.channel_count))
last_channel = self.channels[-1]
for record_index, record in enumerate(data):
record[-1] = 1.0 if record_index >= 100 else 0.0
buffer_server.append(
self.pid,
Record(data=record, timestamp=record_index, rownum=None))
result = buffer_server.query(self.pid,
filters=[(last_channel, ">", 0)],
ordering=("timestamp", "asc"),
max_results=1)
self.assertEqual(len(result), 1)
self.assertEqual(result[0].data[-1], 1.0)
self.assertEqual(result[0].timestamp, 100.0)
def test_multiple_servers(self):
"""Test multiple concurrent servers."""
pid2 = buffer_server.start(self.channels, self._next_buf_name())
n_records = 200
for count, data in enumerate(mock_data(n_records, self.channel_count)):
if count % 2 == 0:
buffer_server.append(self.pid, Record(data, count, None))
else:
buffer_server.append(pid2, Record(data, count, None))
self.assertEqual(buffer_server.count(self.pid), n_records / 2)
self.assertEqual(buffer_server.count(pid2), n_records / 2)
server1_data = buffer_server.get_data(self.pid, 0, 5)
server2_data = buffer_server.get_data(pid2, 0, 5)
self.assertNotEqual(server1_data, server2_data)
buffer_server.stop(pid2)
def test_query_before_flush(self):
"""If a query is made before chunksize records have been written,
the data should still be available."""
n_records = 1000
channel_count = 25
channels = ["ch" + str(c) for c in range(channel_count)]
buf = Buffer(channels=channels, chunksize=10000)
for i, data in enumerate(mock_data(n_records, channel_count)):
timestamp = float(i)
buf.append(Record(data, timestamp, None))
rows = buf.query(start=buf.start_time, field='timestamp')
self.assertEqual(len(rows), n_records)
self.assertEqual(len(buf.all()), n_records)
buf.cleanup()
def test_latest(self):
"""Test query for most recent items."""
n_records = 1000
latest_n = 100
channel_count = 25
channels = ["ch" + str(c) for c in range(channel_count)]
buf = Buffer(channels=channels)
latest = []
for i, data in enumerate(mock_data(n_records, channel_count)):
timestamp = float(i)
if i >= n_records - latest_n:
latest.append((data, timestamp, i+1))
buf.append(Record(data, timestamp, None))
rows = buf.latest(latest_n)
for j, item in enumerate(reversed(latest)):
self.assertEqual(item, rows[j])
buf.cleanup()
def test_buffer(self):
"""Test Buffer functionality."""
n_records = 15000
channel_count = 25
channels = ["ch" + str(c) for c in range(channel_count)]
buf = Buffer(channels=channels, chunksize=10000)
append_timer = _Timer()
timevalues = {}
for i, data in enumerate(mock_data(n_records, channel_count)):
timestamp = float(i)
if i % 1000 == 0:
timevalues[timestamp] = data
with append_timer:
buf.append(Record(data, timestamp, None))
self.assertEqual(buf.start_time, 0.0)
starttime = 0.0
rows = buf.query(start=starttime, end=starttime +
1.0, field='timestamp')
self.assertEqual(len(rows), 1, "Results should not include end value.")
self.assertEqual(rows[0].timestamp, starttime)
self.assertEqual(rows[0].data, timevalues[starttime])
start_time = 1000.0
end_time = 2000.0
rows = buf.query(start=start_time, end=end_time, field='timestamp')
self.assertEqual(len(rows), start_time)
self.assertEqual(rows[0].data, timevalues[start_time])
rows = buf.query(start=buf.start_time, field='timestamp')
self.assertEqual(
len(rows), n_records, ("Providing only the start should return "
"the rest."))
buf.cleanup()
def __init__(self, *args, **kwargs):
super(TestDataAcquistionClient, self).__init__(*args, **kwargs)
num_channels = 25
num_records = 500
self.mock_channels = ['ch' + str(i) for i in range(num_channels)]
self.mock_data = list(mock_data(num_records, num_channels))
