def test_ThreadPollInput():
app = pg.mkQApp()
outstream = OutputStream()
outstream.configure(**stream_spec)
instream = InputStream()
instream.connect(outstream)
sender = ThreadSender(output_stream=outstream)
poller = ThreadPollInput(input_stream=instream)
global last_pos
last_pos = 0
def on_new_data(pos, arr):
assert arr.shape == (chunksize, nb_channel)
global last_pos
last_pos += chunksize
assert last_pos == pos
def terminate():
sender.wait()
poller.stop()
poller.wait()
app.quit()
sender.terminated.connect(terminate)
poller.new_data.connect(on_new_data)
poller.start()
sender.start()
app.exec_()
def check_stream_ringbuffer(**kwds):
chunk_shape = (-1, 16)
stream_spec = dict(protocol='tcp', interface='127.0.0.1', port='*',
transfermode='plaindata', streamtype='analogsignal',
dtype='float32', shape=chunk_shape, compression='',
scale=None, offset=None, units='', axisorder=None,
double=True)
stream_spec.update(kwds)
print(" %s" % kwds)
outstream = OutputStream()
outstream.configure(**stream_spec)
instream = InputStream()
instream.connect(outstream)
instream.set_buffer(stream_spec['buffer_size'], axisorder=stream_spec['axisorder'],
double=stream_spec['double'])
# Make sure we are re-using sharedmem buffer
if instream.receiver.buffer is not None:
assert instream._own_buffer is False
time.sleep(.1)
data = np.random.normal(size=(4096, 16)).astype('float32')
for i in range(16):
chunk = data[i*256:(i+1)*256]
outstream.send(chunk)
instream.recv()
data2 = instream[0:4096]
assert np.all(data2 == data)
if outstream.params['axisorder'] is not None:
assert np.all(np.argsort(data2.strides)[::-1] == outstream.params['axisorder'])
def test_ThreadPollInput():
app = pg.mkQApp()
outstream = OutputStream()
outstream.configure(**stream_spec)
instream = InputStream()
instream.connect(outstream)
sender = ThreadSender(output_stream=outstream)
poller = ThreadPollInput(input_stream=instream)
global last_pos
last_pos= 0
def on_new_data(pos, arr):
assert arr.shape==(chunksize, nb_channel)
global last_pos
last_pos += chunksize
assert last_pos==pos
def terminate():
sender.wait()
poller.stop()
poller.wait()
app.quit()
sender.terminated.connect(terminate)
poller.new_data.connect(on_new_data)
poller.start()
sender.start()
app.exec_()
def test_stream_splitter():
app = pg.mkQApp()
stream_spec = dict(protocol='tcp',
interface='127.0.0.1',
port='*',
transfermode='plaindata',
streamtype='analogsignal',
dtype='float32',
shape=(-1, nb_channel),
timeaxis=0,
compression='',
scale=None,
offset=None,
units='')
outstream = OutputStream()
outstream.configure(**stream_spec)
sender = ThreadSender(output_stream=outstream)
def on_new_data(pos, arr):
assert arr.shape == (chunksize, 1)
# print(pos, arr.shape)
all_instream = []
all_poller = []
splitter = StreamSplitter()
splitter.configure()
splitter.input.connect(outstream)
for name, output in splitter.outputs.items():
output.configure()
instream = InputStream()
instream.connect(output)
poller = ThreadPollInput(input_stream=instream)
poller.new_data.connect(on_new_data)
all_instream.append(instream)
all_poller.append(poller)
splitter.initialize()
def terminate():
sender.wait()
splitter.stop()
for poller in all_poller:
poller.stop()
poller.wait()
app.quit()
sender.terminated.connect(terminate)
for poller in all_poller:
poller.start()
splitter.start()
sender.start()
app.exec_()
def test_streamconverter():
app = pg.mkQApp()
stream_spec = dict(protocol='tcp', interface='127.0.0.1', port='*',
transfermode='plaindata', streamtype='analogsignal',
dtype='float32', shape=(-1, nb_channel), timeaxis = 0,
compression ='', scale = None, offset = None, units = '')
outstream = OutputStream()
outstream.configure(**stream_spec)
sender = ThreadSender(output_stream=outstream)
stream_spec2 = dict(protocol='tcp', interface='127.0.0.1', port='*',
transfermode='sharedarray', streamtype='analogsignal',
dtype='float32', shape=(nb_channel, -1), timeaxis = 1,
compression ='', scale = None, offset = None, units = '',
sharedarray_shape = (nb_channel, chunksize*20), ring_buffer_method = 'double',
)
conv = StreamConverter()
conv.configure()
conv.input.connect(outstream)
conv.output.configure(**stream_spec2)
conv.initialize()
instream = InputStream()
instream.connect(conv.output)
global last_pos
last_pos= 0
def on_new_data(pos, arr):
assert arr is None
global last_pos
last_pos += chunksize
assert last_pos==pos
def terminate():
sender.wait()
conv.stop()
poller.stop()
poller.wait()
app.quit()
poller = ThreadPollInput(input_stream=instream)
sender.terminated.connect(terminate)
poller.new_data.connect(on_new_data)
poller.start()
conv.start()
sender.start()
app.exec_()
def test_stream_splitter():
app = pg.mkQApp()
stream_spec = dict(protocol='tcp', interface='127.0.0.1', port='*',
transfermode='plaindata', streamtype='analogsignal',
dtype='float32', shape=(-1, nb_channel), timeaxis = 0,
compression ='', scale = None, offset = None, units = '')
outstream = OutputStream()
outstream.configure(**stream_spec)
sender = ThreadSender(output_stream=outstream)
def on_new_data(pos, arr):
assert arr.shape==(chunksize, 1)
# print(pos, arr.shape)
all_instream = []
all_poller = []
splitter = StreamSplitter()
splitter.configure()
splitter.input.connect(outstream)
for name, output in splitter.outputs.items():
output.configure()
instream = InputStream()
instream.connect(output)
poller = ThreadPollInput(input_stream=instream)
poller.new_data.connect(on_new_data)
all_instream.append(instream)
all_poller.append(poller)
splitter.initialize()
def terminate():
sender.wait()
splitter.stop()
for poller in all_poller:
poller.stop()
poller.wait()
app.quit()
sender.terminated.connect(terminate)
for poller in all_poller:
poller.start()
splitter.start()
sender.start()
app.exec_()
def check_stream(chunksize=1024, chan_shape=(16,), **kwds):
chunk_shape = (chunksize,) + chan_shape
stream_spec = dict(protocol='tcp', interface='127.0.0.1', port='*',
transfermode='plaindata', streamtype='analogsignal',
dtype='float32', shape=chunk_shape, compression='',
scale=None, offset=None, units='')
stream_spec.update(kwds)
print(" %s" % kwds)
outstream = OutputStream()
outstream.configure(**stream_spec)
instream = InputStream()
instream.connect(outstream)
time.sleep(.1)
for i in range(5):
#~ print(i)
# send
if i == 1:
# send non-aligned data
cs = (chunk_shape[1], chunk_shape[0]) + chunk_shape[2:]
arr = np.random.rand(*cs).transpose(1,0).astype(stream_spec['dtype'])
else:
arr = np.random.rand(*chunk_shape).astype(stream_spec['dtype'])
outstream.send(arr)
# recv
index, arr2 = instream.recv(return_data=True)
assert index == outstream.last_index
assert np.all((arr-arr2)==0.)
outstream.close()
instream.close()
def test_stream_plaindata():
nb_channel = 16
chunksize = 1024
stream_spec = dict(protocol = 'tcp', interface = '127.0.0.1', port='*',
transfermode = 'plaindata', streamtype = 'analogsignal',
dtype = 'float32', shape = (-1, nb_channel), compression ='',
scale = None, offset = None, units = '')
for protocol in protocols:
for compression in compressions:
print(protocol, compression)
stream_spec['protocol'] = protocol
stream_spec['compression'] = compression
outstream = OutputStream()
outstream.configure(**stream_spec)
instream = InputStream()
instream.connect(outstream)
time.sleep(.5)
index = 0
for i in range(5):
#~ print(i)
#send
index += chunksize
arr = np.random.rand(1024, nb_channel).astype(stream_spec['dtype'])
outstream.send(index, arr)
#recv
index2, arr2 = instream.recv()
assert index2==index
assert np.all((arr-arr2)==0.)
outstream.close()
instream.close()
def test_stream_sharedarray():
# this test is perform with no autoswapaxes
#~ nb_channel = 16
nb_channel = 1
chunksize = 1024
ring_size = chunksize * 5 - 334
stream_spec = dict(protocol='tcp', interface='127.0.0.1', port='*',
transfermode='sharedarray', streamtype='analogsignal',
dtype='float32', shape=(-1, nb_channel), timeaxis = 0, compression ='',
scale = None, offset = None, units = '',
sharedarray_shape = (ring_size, nb_channel), ring_buffer_method= 'single',
)
protocol = 'tcp'
for ring_buffer_method in['single', 'double',]:
for timeaxis in [0, 1]:
print('sharedarray', ring_buffer_method, 'timeaxis', timeaxis)
stream_spec['ring_buffer_method'] = ring_buffer_method
stream_spec['timeaxis'] = timeaxis
if timeaxis == 0:
stream_spec['shape'] = (-1, nb_channel)
stream_spec['sharedarray_shape'] = (ring_size, nb_channel)
elif timeaxis == 1:
stream_spec['shape'] = (nb_channel, -1)
stream_spec['sharedarray_shape'] = (nb_channel, ring_size)
outstream = OutputStream()
outstream.configure(**stream_spec)
instream = InputStream()
instream.connect(outstream)
time.sleep(.5)
index = 0
for i in range(30):
#~ print(i)
# send
if timeaxis==0:
arr = np.tile(np.arange(index, index+chunksize)[:, None], (1,nb_channel)).astype(stream_spec['dtype'])
elif timeaxis==1:
arr = np.tile(np.arange(index, index+chunksize)[None,:], (nb_channel, 1)).astype(stream_spec['dtype'])
index += chunksize
outstream.send(index, arr, autoswapaxes=False)
index2, arr2 = instream.recv(autoswapaxes=False, with_data = False)
assert index2==index
assert arr2 is None
# get a buffer of size chunksize*3
if ring_buffer_method == 'double' and index>chunksize*3:
arr2 = instream.get_array_slice(index2, chunksize*3, autoswapaxes=False)
if timeaxis==0:
assert np.all(arr2[:,0]==np.arange(index-chunksize*3, index).astype('float32'))
elif timeaxis==1:
assert np.all(arr2[0,:]==np.arange(index-chunksize*3, index).astype('float32'))
outstream.close()
instream.close()
def test_stream_plaindata():
nb_channel = 16
chunksize = 1024
stream_spec = dict(protocol='tcp',
interface='127.0.0.1',
port='*',
transfermode='plaindata',
streamtype='analogsignal',
dtype='float32',
shape=(-1, nb_channel),
compression='',
scale=None,
offset=None,
units='')
for protocol in protocols:
for compression in compressions:
print(protocol, compression)
stream_spec['protocol'] = protocol
stream_spec['compression'] = compression
outstream = OutputStream()
outstream.configure(**stream_spec)
instream = InputStream()
instream.connect(outstream)
time.sleep(.5)
index = 0
for i in range(5):
#~ print(i)
# send
index += chunksize
arr = np.random.rand(1024,
nb_channel).astype(stream_spec['dtype'])
outstream.send(index, arr, autoswapaxes=False)
# recv
index2, arr2 = instream.recv(autoswapaxes=False)
assert index2 == index
assert np.all((arr - arr2) == 0.)
outstream.close()
instream.close()
def check_stream(chunksize=1024, chan_shape=(16, ), **kwds):
chunk_shape = (chunksize, ) + chan_shape
stream_spec = dict(protocol='tcp',
interface='127.0.0.1',
port='*',
transfermode='plaindata',
streamtype='analogsignal',
dtype='float32',
shape=chunk_shape,
compression='',
scale=None,
offset=None,
units='')
stream_spec.update(kwds)
print(" %s" % kwds)
outstream = OutputStream()
outstream.configure(**stream_spec)
instream = InputStream()
instream.connect(outstream)
time.sleep(.1)
for i in range(5):
#~ print(i)
# send
if i == 1:
# send non-aligned data
cs = (chunk_shape[1], chunk_shape[0]) + chunk_shape[2:]
arr = np.random.rand(*cs).transpose(1,
0).astype(stream_spec['dtype'])
else:
arr = np.random.rand(*chunk_shape).astype(stream_spec['dtype'])
outstream.send(arr)
# recv
index, arr2 = instream.recv(return_data=True)
assert index == outstream.last_index
assert np.all((arr - arr2) == 0.)
outstream.close()
instream.close()
def benchmark_stream(protocol, transfermode, compression, chunksize, nb_channels=16, nloop=10, profile=False):
ring_size = chunksize*20
stream_spec = dict(protocol=protocol, interface='127.0.0.1', port='*',
transfermode=transfermode, streamtype = 'analogsignal',
dtype='float32', shape=(-1, nb_channels), compression=compression,
scale=None, offset=None, units='',
# for sharedarray
sharedarray_shape = ( ring_size, nb_channels), timeaxis = 0,
ring_buffer_method = 'double',
)
outstream = OutputStream()
outstream.configure(**stream_spec)
time.sleep(.5)
instream = InputStream()
instream.connect(outstream)
arr = np.random.rand(chunksize, nb_channels).astype(stream_spec['dtype'])
perf = []
prof = cProfile.Profile()
for i in range(nloop):
start = time.perf_counter()
if profile:
prof.enable()
outstream.send(arr)
index2, arr2 = instream.recv()
if profile:
prof.disable()
perf.append(time.perf_counter() - start)
if profile:
prof.print_stats('cumulative')
outstream.close()
instream.close()
dt = np.min(perf)
print(chunksize, nloop, transfermode, protocol.ljust(6), compression.ljust(13), 'time = %0.02f ms' % (dt*1000), 'speed = ', chunksize*nb_channels*4*1e-6/dt, 'MB/s')
return dt
def check_stream_struct_dtype(**kwds):
dtype_struct = [
('datetime', 'datetime64[ms]'),
('message', 'S32'),
]
chunk_shape = (-1, )
stream_spec = dict(protocol='tcp',
interface='127.0.0.1',
port='*',
transfermode='plaindata',
streamtype='analogsignal',
dtype=dtype_struct,
shape=chunk_shape,
compression='',
scale=None,
offset=None,
units='',
axisorder=None,
double=True)
stream_spec.update(kwds)
print(" %s" % kwds)
outstream = OutputStream()
outstream.configure(**stream_spec)
instream = InputStream()
instream.connect(outstream)
instream.set_buffer(stream_spec['buffer_size'],
axisorder=stream_spec['axisorder'],
double=stream_spec['double'])
# Make sure we are re-using sharedmem buffer
if instream.receiver.buffer is not None:
assert instream._own_buffer is False
time.sleep(.1)
data = np.zeros(50, dtype=dtype_struct)
for i in range(10):
chunk = data[i * 5:(i + 1) * 5]
outstream.send(chunk)
instream.recv()
data2 = instream[0:50]
assert np.all(data2 == data)
if outstream.params['axisorder'] is not None:
assert np.all(
np.argsort(data2.strides)[::-1] == outstream.params['axisorder'])
def check_stream_ringbuffer(**kwds):
chunk_shape = (-1, 16)
stream_spec = dict(protocol='tcp',
interface='127.0.0.1',
port='*',
transfermode='plaindata',
streamtype='analogsignal',
dtype='float32',
shape=chunk_shape,
compression='',
scale=None,
offset=None,
units='',
axisorder=None,
double=True)
stream_spec.update(kwds)
print(" %s" % kwds)
outstream = OutputStream()
outstream.configure(**stream_spec)
instream = InputStream()
instream.connect(outstream)
instream.set_buffer(stream_spec['buffer_size'],
axisorder=stream_spec['axisorder'],
double=stream_spec['double'])
# Make sure we are re-using sharedmem buffer
if instream.receiver.buffer is not None:
assert instream._own_buffer is False
time.sleep(.1)
data = np.random.normal(size=(4096, 16)).astype('float32')
for i in range(16):
chunk = data[i * 256:(i + 1) * 256]
outstream.send(chunk)
instream.recv()
data2 = instream[0:4096]
assert np.all(data2 == data)
if outstream.params['axisorder'] is not None:
assert np.all(
np.argsort(data2.strides)[::-1] == outstream.params['axisorder'])
def test_streamconverter():
app = pg.mkQApp()
stream_spec = dict(protocol='tcp',
interface='127.0.0.1',
port='*',
transfermode='plaindata',
streamtype='analogsignal',
dtype='float32',
shape=(-1, nb_channel),
timeaxis=0,
compression='',
scale=None,
offset=None,
units='')
outstream = OutputStream()
outstream.configure(**stream_spec)
sender = ThreadSender(output_stream=outstream)
stream_spec2 = dict(
protocol='tcp',
interface='127.0.0.1',
port='*',
transfermode='sharedarray',
streamtype='analogsignal',
dtype='float32',
shape=(nb_channel, -1),
timeaxis=1,
compression='',
scale=None,
offset=None,
units='',
sharedarray_shape=(nb_channel, chunksize * 20),
ring_buffer_method='double',
)
conv = StreamConverter()
conv.configure()
conv.input.connect(outstream)
conv.output.configure(**stream_spec2)
conv.initialize()
instream = InputStream()
instream.connect(conv.output)
global last_pos
last_pos = 0
def on_new_data(pos, arr):
assert arr is None
global last_pos
last_pos += chunksize
assert last_pos == pos
def terminate():
sender.wait()
conv.stop()
poller.stop()
poller.wait()
app.quit()
poller = ThreadPollInput(input_stream=instream)
sender.terminated.connect(terminate)
poller.new_data.connect(on_new_data)
poller.start()
conv.start()
sender.start()
app.exec_()
def test_autoswapaxes():
# the recv shape is alwas (10,2)
nb_channel = 2
l = 10
data = np.empty((10, nb_channel), dtype='float32')
assert data.flags['C_CONTIGUOUS']
# timeaxis 0 / plaindata
outstream, instream = OutputStream(), InputStream()
outstream.configure(transfermode='plaindata',
timeaxis=0,
shape=(-1, nb_channel))
instream.connect(outstream)
time.sleep(.5)
outstream.send(l, data)
pos, data2 = instream.recv()
assert data2.shape == (l, 2)
assert data2.flags['C_CONTIGUOUS']
# timeaxis 1 / plaindata
outstream, instream = OutputStream(), InputStream()
outstream.configure(transfermode='plaindata',
timeaxis=1,
shape=(nb_channel, -1))
instream.connect(outstream)
time.sleep(.5)
outstream.send(l, data)
pos, data2 = instream.recv()
assert data2.shape == (l, 2)
assert not data2.flags['C_CONTIGUOUS']
# timeaxis 0 / sharedarray
outstream, instream = OutputStream(), InputStream()
outstream.configure(transfermode='sharedarray',
timeaxis=0,
shape=(-1, nb_channel),
sharedarray_shape=(l * 5, nb_channel),
ring_buffer_method='double')
instream.connect(outstream)
assert instream.receiver._numpyarr.flags['C_CONTIGUOUS']
assert not instream.receiver._numpyarr[:, 0].flags['C_CONTIGUOUS']
time.sleep(.5)
outstream.send(l, data)
pos, data2 = instream.recv(with_data=True)
assert data2.flags['C_CONTIGUOUS']
assert data2.shape == (l, 2)
# timeaxis 1 / sharedarray
outstream, instream = OutputStream(), InputStream()
outstream.configure(transfermode='sharedarray',
timeaxis=1,
shape=(nb_channel, -1),
sharedarray_shape=(nb_channel, l * 5),
ring_buffer_method='double')
instream.connect(outstream)
assert instream.receiver._numpyarr.flags['C_CONTIGUOUS']
assert instream.receiver._numpyarr[0, :].flags['C_CONTIGUOUS']
time.sleep(.5)
outstream.send(l, data)
pos, data2 = instream.recv(with_data=True)
assert not data2.flags['C_CONTIGUOUS']
assert data2.shape == (l, 2)
def test_autoswapaxes():
# the recv shape is alwas (10,2)
nb_channel = 2
l = 10
data = np.empty((10, nb_channel), dtype = 'float32')
assert data.flags['C_CONTIGUOUS']
# timeaxis 0 / plaindata
outstream, instream = OutputStream(),InputStream()
outstream.configure(transfermode = 'plaindata', timeaxis=0, shape = (-1,nb_channel))
instream.connect(outstream)
time.sleep(.5)
outstream.send(l, data)
pos, data2 = instream.recv()
assert data2.shape == (l,2)
assert data2.flags['C_CONTIGUOUS']
# timeaxis 1 / plaindata
outstream, instream = OutputStream(),InputStream()
outstream.configure(transfermode = 'plaindata', timeaxis=1, shape = (nb_channel, -1))
instream.connect(outstream)
time.sleep(.5)
outstream.send(l, data)
pos, data2 = instream.recv()
assert data2.shape == (l,2)
assert not data2.flags['C_CONTIGUOUS']
# timeaxis 0 / sharedarray
outstream, instream = OutputStream(),InputStream()
outstream.configure(transfermode = 'sharedarray', timeaxis=0, shape = (-1,nb_channel),
sharedarray_shape = (l*5, nb_channel), ring_buffer_method = 'double')
instream.connect(outstream)
assert instream.receiver._numpyarr.flags['C_CONTIGUOUS']
assert not instream.receiver._numpyarr[:, 0].flags['C_CONTIGUOUS']
time.sleep(.5)
outstream.send(l, data)
pos, data2 = instream.recv(with_data = True)
assert data2.flags['C_CONTIGUOUS']
assert data2.shape == (l,2)
# timeaxis 1 / sharedarray
outstream, instream = OutputStream(),InputStream()
outstream.configure(transfermode = 'sharedarray', timeaxis=1, shape = (nb_channel, -1),
sharedarray_shape = (nb_channel, l*5), ring_buffer_method = 'double')
instream.connect(outstream)
assert instream.receiver._numpyarr.flags['C_CONTIGUOUS']
assert instream.receiver._numpyarr[0, :].flags['C_CONTIGUOUS']
time.sleep(.5)
outstream.send(l, data)
pos, data2 = instream.recv(with_data = True)
assert not data2.flags['C_CONTIGUOUS']
assert data2.shape == (l,2)
def test_stream_sharedarray():
# this test is perform with no autoswapaxes
#~ nb_channel = 16
nb_channel = 1
chunksize = 1024
ring_size = chunksize * 5 - 334
stream_spec = dict(
protocol='tcp',
interface='127.0.0.1',
port='*',
transfermode='sharedarray',
streamtype='analogsignal',
dtype='float32',
shape=(-1, nb_channel),
timeaxis=0,
compression='',
scale=None,
offset=None,
units='',
sharedarray_shape=(ring_size, nb_channel),
ring_buffer_method='single',
)
protocol = 'tcp'
for ring_buffer_method in [
'single',
'double',
]:
for timeaxis in [0, 1]:
print('sharedarray', ring_buffer_method, 'timeaxis', timeaxis)
stream_spec['ring_buffer_method'] = ring_buffer_method
stream_spec['timeaxis'] = timeaxis
if timeaxis == 0:
stream_spec['shape'] = (-1, nb_channel)
stream_spec['sharedarray_shape'] = (ring_size, nb_channel)
elif timeaxis == 1:
stream_spec['shape'] = (nb_channel, -1)
stream_spec['sharedarray_shape'] = (nb_channel, ring_size)
outstream = OutputStream()
outstream.configure(**stream_spec)
instream = InputStream()
instream.connect(outstream)
time.sleep(.5)
index = 0
for i in range(30):
#~ print(i)
# send
if timeaxis == 0:
arr = np.tile(
np.arange(index, index + chunksize)[:, None],
(1, nb_channel)).astype(stream_spec['dtype'])
elif timeaxis == 1:
arr = np.tile(
np.arange(index, index + chunksize)[None, :],
(nb_channel, 1)).astype(stream_spec['dtype'])
index += chunksize
outstream.send(index, arr, autoswapaxes=False)
index2, arr2 = instream.recv(autoswapaxes=False,
with_data=False)
assert index2 == index
assert arr2 is None
# get a buffer of size chunksize*3
if ring_buffer_method == 'double' and index > chunksize * 3:
arr2 = instream.get_array_slice(index2,
chunksize * 3,
autoswapaxes=False)
if timeaxis == 0:
assert np.all(
arr2[:,
0] == np.arange(index - chunksize *
3, index).astype('float32'))
elif timeaxis == 1:
assert np.all(arr2[0, :] == np.arange(
index - chunksize * 3, index).astype('float32'))
outstream.close()
instream.close()