def test_send_error(self):
"""An error in sending must be reported."""
# Create a stream with a packet size that is bigger than the likely
# MTU. It should cause an error.
stream = send.UdpStream(spead2.ThreadPool(),
"localhost",
8888,
send.StreamConfig(max_packet_size=100000),
buffer_size=0)
assert_raises(IOError, stream.send_heap, self.heap)
def main(mcast_groups, interface, nheaps):
thread_pool = spead2.ThreadPool(threads=4)
stream = spead2.recv.Stream(thread_pool, spead2.BUG_COMPAT_PYSPEAD_0_5_2)
del thread_pool
pool = spead2.MemoryPool(16384, 26214400, 64, 32)
stream.set_memory_allocator(pool)
for group in mcast_groups:
stream.add_udp_reader(group, 7147, interface_address=interface)
manager = StreamManager(stream)
manager.capture(nheaps)
def test_overlapping_memory_regions(self):
data = memoryview(bytearray(10))
part1 = data[:6]
part2 = data[5:]
config = send.StreamConfig()
udp_ibv_config = send.UdpIbvConfig(
endpoints=[('239.255.88.88', 8888)],
interface_address='10.0.0.1',
memory_regions=[part1, part2])
with pytest.raises(ValueError, match='memory regions overlap'):
send.UdpIbvStream(spead2.ThreadPool(), config, udp_ibv_config)
def setup(self):
self.receiver = spead2.recv.Stream(spead2.ThreadPool())
recv_sock = socket.socket()
recv_sock.bind(("127.0.0.1", 0))
recv_sock.listen(1)
port = recv_sock.getsockname()[1]
self.receiver.add_tcp_reader(acceptor=recv_sock)
recv_sock.close()
self.send_sock = socket.socket()
self.send_sock.connect(("127.0.0.1", port))
def setup(self):
# Make a stream slow enough that we can test async interactions
config = spead2.send.StreamConfig(rate=5e6)
self.stream = UdpStream(spead2.ThreadPool(), 'localhost', 8888, config)
self.ig = spead2.send.ItemGroup()
self.ig.add_item(0x1000,
'test',
'Test item',
shape=(256 * 1024, ),
dtype=np.uint8)
self.ig['test'].value = np.zeros((256 * 1024, ), np.uint8)
self.heap = self.ig.get_heap()
def data_to_heaps(self, data, **kwargs):
"""Take some data and pass it through the receiver to obtain a set of heaps.
Keyword arguments are passed to the receiver constructor.
"""
thread_pool = spead2.ThreadPool()
stop_on_stop_item = kwargs.pop('stop_on_stop_item', None)
stream = recv.Stream(thread_pool, self.flavour.bug_compat, **kwargs)
if stop_on_stop_item is not None:
stream.stop_on_stop_item = stop_on_stop_item
stream.add_buffer_reader(data)
return list(stream)
def test_send_error(self):
"""An error in sending must be reported through the future."""
# Create a stream with a packet size that is bigger than the likely
# MTU. It should cause an error.
stream = UdpStream(spead2.ThreadPool(),
"localhost",
8888,
spead2.send.StreamConfig(max_packet_size=100000),
buffer_size=0)
future = stream.async_send_heap(self.heap)
trollius.get_event_loop().run_until_complete(
self._test_send_error(future))
def setup(self):
# A slow stream, so that we can test overflowing the queue
self.flavour = Flavour(4, 64, 48, 0)
self.stream = send.BytesStream(
spead2.ThreadPool(),
send.StreamConfig(rate=1e6, max_heaps=2))
# A large heap
ig = send.ItemGroup(flavour=self.flavour)
ig.add_item(0x1000, 'test', 'A large item', shape=(256 * 1024,), dtype=np.uint8)
ig['test'].value = np.zeros((256 * 1024,), np.uint8)
self.heap = ig.get_heap()
self.threads = []
def test_packet_presence(self, data_ring, queue):
"""Test packet presence feature."""
# Each heap is split into two packets. Create a free ring where the
# chunks have space for this.
free_ring = spead2.recv.ChunkRingbuffer(4)
while not free_ring.full():
free_ring.put(
recv.Chunk(present=np.zeros(HEAPS_PER_CHUNK * PACKETS_PER_HEAP,
np.uint8),
data=np.zeros(CHUNK_PAYLOAD_SIZE, np.uint8)))
stream_config = spead2.recv.StreamConfig(max_heaps=128,
allow_out_of_order=True)
# Note: user_data is deliberately not assigned to a local variable, so
# that reference-counting errors are more likely to be detected.
user_data = np.zeros(1, dtype=user_data_type.dtype)
user_data["scale"] = PACKETS_PER_HEAP
user_data["placed_heaps_index"] = stream_config.add_stat(
"placed_heaps")
place_bind_llc = scipy.LowLevelCallable(
place_bind.ctypes,
user_data=user_data.ctypes.data_as(ctypes.c_void_p),
signature='void (void *, size_t, void *)')
stream = spead2.recv.ChunkRingStream(
spead2.ThreadPool(), stream_config,
spead2.recv.ChunkStreamConfig(
items=[
0x1000, spead2.HEAP_LENGTH_ID, spead2.PAYLOAD_LENGTH_ID
],
max_chunks=4,
place=place_bind_llc,
).enable_packet_presence(PACKET_SIZE), data_ring, free_ring)
stream.add_inproc_reader(queue)
queue.add_packet(self.make_packet(4, 0, PACKET_SIZE))
queue.add_packet(self.make_packet(4, PACKET_SIZE, 2 * PACKET_SIZE))
queue.add_packet(self.make_packet(17, PACKET_SIZE, 2 * PACKET_SIZE))
queue.stop()
chunks = list(stream.data_ringbuffer)
assert len(chunks) == 2
self.check_chunk_packets(
chunks[0], 0,
np.array(
[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
np.uint8))
self.check_chunk_packets(
chunks[1], 1,
np.array(
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0],
np.uint8))
assert stream.stats["placed_heaps"] == 2
def recv_stream(self, data_ring, free_ring, queue):
stream = spead2.recv.ChunkRingStream(
spead2.ThreadPool(),
# max_heaps is artificially high to make test_packet_too_old work
spead2.recv.StreamConfig(max_heaps=128),
spead2.recv.ChunkStreamConfig(
items=[0x1000, spead2.HEAP_LENGTH_ID],
max_chunks=4,
place=place_plain_llc),
data_ring,
free_ring)
stream.add_inproc_reader(queue)
yield stream
stream.stop()
def transmit_item_group(self, item_group, memcpy, allocator):
thread_pool = spead2.ThreadPool(2)
sender = spead2.send.BytesStream(thread_pool)
gen = spead2.send.HeapGenerator(item_group)
sender.send_heap(gen.get_heap())
sender.send_heap(gen.get_end())
receiver = spead2.recv.Stream(thread_pool)
if allocator is not None:
receiver.set_memory_allocator(allocator)
receiver.set_memcpy(memcpy)
receiver.add_buffer_reader(sender.getvalue())
received_item_group = spead2.ItemGroup()
for heap in receiver:
received_item_group.update(heap)
return received_item_group
def initialize(self, **kwargs):
BarrierAppDROP.initialize(self, **kwargs)
self._port = self._getArg(kwargs, 'port', None)
if self._port is None:
raise InvalidDropException(self, 'port is not set')
self._stream = spead2.recv.Stream(spead2.ThreadPool(), 0)
self._stream.add_udp_reader(int(self._port))
try:
ip = dlg.utils.get_local_ip_addr()[0][0]
con = httplib.HTTPConnection('sdp-dfms.ddns.net', 8096)
con.request('GET', '/reg_receiver?ip=%s' % (ip))
except:
logger.exception('Failed to register, will continue anyway')
async def test_async_iter(self):
tp = spead2.ThreadPool()
queue = spead2.InprocQueue()
sender = spead2.send.InprocStream(tp, queue)
ig = spead2.send.ItemGroup()
sender.send_heap(ig.get_start())
sender.send_heap(ig.get_end())
queue.stop()
heaps = []
receiver = spead2.recv.asyncio.Stream(tp)
receiver.stop_on_stop_item = False
receiver.add_inproc_reader(queue)
async for heap in receiver:
heaps.append(heap)
assert_equal(2, len(heaps))
assert_true(heaps[0].is_start_of_stream())
assert_true(heaps[1].is_end_of_stream())
def transmit_item_group(self, item_group, memcpy, allocator):
thread_pool = spead2.ThreadPool(2)
sender = spead2.send.UdpStream(
thread_pool, "localhost", 8888,
spead2.send.StreamConfig(rate=1e8),
buffer_size=0)
receiver = spead2.recv.Stream(thread_pool)
receiver.set_memcpy(memcpy)
if allocator is not None:
receiver.set_memory_allocator(allocator)
receiver.add_udp_reader(8888, bind_hostname="localhost")
gen = spead2.send.HeapGenerator(item_group)
sender.send_heap(gen.get_heap())
sender.send_heap(gen.get_end())
received_item_group = spead2.ItemGroup()
for heap in receiver:
received_item_group.update(heap)
return received_item_group
def __init__(self, debug=False):
self.thread_pool = spead2.ThreadPool()
self.stream = spead2.recv.Stream(self.thread_pool)
self.pool = spead2.MemoryPool(16384, 26214400, 12, 8)
self.debug = debug
self.metadata = None
self.stream.set_memory_allocator(self.pool)
self.stream.add_udp_reader(9021, max_size=10000, buffer_size=10000)
if self.debug:
logging.basicConfig(level=logging.DEBUG,
format='%(levelname)s %(asctime)s %(message)s')
else:
logging.basicConfig(filename='receive.log',
level=logging.DEBUG,
format='%(levelname)s %(asctime)s %(message)s')
def transmit_item_group(self, item_group, memcpy, allocator):
"""Transmit `item_group` over the chosen transport,
and return the item group received at the other end. Subclasses
should override this.
"""
thread_pool = spead2.ThreadPool(2)
receiver = spead2.recv.Stream(thread_pool)
receiver.set_memcpy(memcpy)
if allocator is not None:
receiver.set_memory_allocator(allocator)
self.prepare_receiver(receiver)
sender = self.prepare_sender(thread_pool)
gen = spead2.send.HeapGenerator(item_group)
sender.send_heap(gen.get_heap())
sender.send_heap(gen.get_end())
received_item_group = spead2.ItemGroup()
for heap in receiver:
received_item_group.update(heap)
return received_item_group
def test_no_stop_heap(self):
"""A heap containing a stop is not passed to the ring"""
thread_pool = spead2.ThreadPool(1)
sender = send.BytesStream(thread_pool)
ig = send.ItemGroup()
data = np.array([[6, 7, 8], [10, 11, 12000]], dtype=np.uint16)
ig.add_item(id=0x2345,
name='name',
description='description',
shape=data.shape,
dtype=data.dtype,
value=data)
gen = send.HeapGenerator(ig)
sender.send_heap(gen.get_heap(data='all'))
sender.send_heap(gen.get_end())
receiver = spead2.recv.Stream(thread_pool)
receiver.add_buffer_reader(sender.getvalue())
heaps = list(receiver)
assert_equal(1, len(heaps))
def test_full_stop(self):
"""Must be able to stop even if the consumer is not consuming
anything."""
thread_pool = spead2.ThreadPool(1)
sender = send.BytesStream(thread_pool)
ig = send.ItemGroup()
data = np.array([[6, 7, 8], [10, 11, 12000]], dtype=np.uint16)
ig.add_item(id=0x2345,
name='name',
description='description',
shape=data.shape,
dtype=data.dtype,
value=data)
gen = send.HeapGenerator(ig)
for i in range(10):
sender.send_heap(gen.get_heap(data='all'))
receiver = spead2.recv.Stream(thread_pool)
receiver.add_buffer_reader(sender.getvalue())
receiver.stop()
def __init__(self,
mc_ip,
mc_port,
capture_ip,
speadhandler,
numa_affinity=[]):
"""
Args:
mc_ip: Array of multicast group IPs
mc_port: Port number of multicast streams
capture_ip: Interface to capture streams from MCG on
handler: Object that handles data in the stream
"""
Thread.__init__(self, name=self.__class__.__name__)
self._mc_ip = mc_ip
self._mc_port = mc_port
self._capture_ip = capture_ip
self._stop_event = Event()
self._handler = speadhandler
#ToDo: fix magic numbers for parameters in spead stream
thread_pool = spead2.ThreadPool(
threads=8, affinity=[int(k) for k in numa_affinity])
self.stream = spead2.recv.Stream(thread_pool,
spead2.BUG_COMPAT_PYSPEAD_0_5_2,
max_heaps=64,
ring_heaps=64,
contiguous_only=False)
pool = spead2.MemoryPool(16384, ((32 * 4 * 1024**2) + 1024),
max_free=64,
initial=64)
self.stream.set_memory_allocator(pool)
##ToDo: fix magic numbers for parameters in spead stream
#thread_pool = spead2.ThreadPool(threads=4)
#self.stream = spead2.recv.Stream(thread_pool, spead2.BUG_COMPAT_PYSPEAD_0_5_2, max_heaps=64, ring_heaps=64, contiguous_only = False)
#pool = spead2.MemoryPool(16384, ((32*4*1024**2)+1024), max_free=64, initial=64)
#self.stream.set_memory_allocator(pool)
#self.rb = self.stream.ringbuffer
self._nheaps = 0
self._incomplete_heaps = 0
self._complete_heaps = 0
def transmit_item_group(self, item_group, memcpy, allocator):
thread_pool = spead2.ThreadPool(2)
mcast_group = '239.255.88.88'
interface_address = '127.0.0.1'
sender = spead2.send.UdpStream(
thread_pool, mcast_group, 8887,
spead2.send.StreamConfig(rate=1e8),
buffer_size=0, ttl=1, interface_address=interface_address)
receiver = spead2.recv.Stream(thread_pool)
receiver.set_memcpy(memcpy)
if allocator is not None:
receiver.set_memory_allocator(allocator)
receiver.add_udp_reader(mcast_group, 8887, interface_address=interface_address)
gen = spead2.send.HeapGenerator(item_group)
sender.send_heap(gen.get_heap())
sender.send_heap(gen.get_end())
received_item_group = spead2.ItemGroup()
for heap in receiver:
received_item_group.update(heap)
return received_item_group
def __init__(self,
mc_ip,
mc_port,
capture_ip,
spead_handler,
package_handler,
numa_affinity=[]):
"""
Args:
mc_ip: Array of multicast group IPs
mc_port: Port number of multicast streams
capture_ip: Interface to capture streams from MCG on
handler: Object that handles data in the stream
"""
Thread.__init__(self, name=self.__class__.__name__)
self._mc_ip = mc_ip
self._mc_port = mc_port
self._capture_ip = capture_ip
self._stop_event = Event()
self._spead_handler = spead_handler
self._package_handler = package_handler
#ToDo: fix magic numbers for parameters in spead stream
thread_pool = spead2.ThreadPool(
threads=8, affinity=[int(k) for k in numa_affinity])
pool = spead2.MemoryPool(lower=16384,
upper=((32 * 4 * 1024**2) + 1024),
max_free=64,
initial=64)
stream_config = spead2.recv.StreamConfig(
bug_compat=spead2.BUG_COMPAT_PYSPEAD_0_5_2,
max_heaps=64,
memory_allocator=pool)
ring_stream_config = spead2.recv.RingStreamConfig(
heaps=64, contiguous_only=False)
self.stream = spead2.recv.Stream(thread_pool, stream_config,
ring_stream_config)
self._nheaps = 0
self._incomplete_heaps = 0
self._complete_heaps = 0
def _create_streams(config, log):
"""Construct streams, item group and item descriptions."""
# bug_compat = spead2.BUG_COMPAT_PYSPEAD_0_5_2
bug_compat = 0
lower = config['memory_pool']['lower']
upper = config['memory_pool']['upper']
max_free = config['memory_pool']['max_free']
initial = config['memory_pool']['initial']
#streams = []
for stream_data in config['streams']:
log.debug('Creating stream on port {}'.format(stream_data['port']))
#s = spead2.recv.Stream(spead2.ThreadPool(), bug_compat)
# Async Receive
stream = spead2.recv.asyncio.Stream(spead2.ThreadPool(), bug_compat)
pool = spead2.MemoryPool(lower, upper, max_free, initial)
stream.set_memory_allocator(pool)
stream.add_udp_reader(stream_data['port'])
# streams.append(s)
return stream
def transmit_item_group(self, item_group, memcpy, allocator):
self.check_ipv6()
interface_index = self.get_interface_index()
thread_pool = spead2.ThreadPool(2)
mcast_group = 'ff14::1234'
sender = spead2.send.UdpStream(
thread_pool, mcast_group, 8887,
spead2.send.StreamConfig(rate=1e8),
buffer_size=0, ttl=0, interface_index=interface_index)
receiver = spead2.recv.Stream(thread_pool)
receiver.set_memcpy(memcpy)
if allocator is not None:
receiver.set_memory_allocator(allocator)
receiver.add_udp_reader(mcast_group, 8887, interface_index=interface_index)
gen = spead2.send.HeapGenerator(item_group)
sender.send_heap(gen.get_heap())
sender.send_heap(gen.get_end())
received_item_group = spead2.ItemGroup()
for heap in receiver:
received_item_group.update(heap)
return received_item_group
def transmit_item_group_async(self, item_group, memcpy, allocator):
thread_pool = spead2.ThreadPool(2)
receiver = spead2.recv.trollius.Stream(thread_pool, loop=self.loop)
receiver.set_memcpy(memcpy)
if allocator is not None:
receiver.set_memory_allocator(allocator)
yield From(self.prepare_receiver(receiver))
sender = yield From(self.prepare_sender(thread_pool))
gen = spead2.send.HeapGenerator(item_group)
yield From(sender.async_send_heap(gen.get_heap()))
yield From(sender.async_send_heap(gen.get_end()))
yield From(sender.async_flush())
received_item_group = spead2.ItemGroup()
while True:
try:
heap = yield From(receiver.get())
except spead2.Stopped:
break
else:
received_item_group.update(heap)
raise Return(received_item_group)
def transmit_item_group(self, item_group, memcpy, allocator):
thread_pool = spead2.ThreadPool(2)
send_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
recv_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sender = spead2.send.UdpStream(
thread_pool, "127.0.0.1", 8888,
spead2.send.StreamConfig(rate=1e8),
buffer_size=0, socket=send_sock)
receiver = spead2.recv.Stream(thread_pool)
receiver.set_memcpy(memcpy)
if allocator is not None:
receiver.set_memory_allocator(allocator)
receiver.add_udp_reader(8888, bind_hostname="127.0.0.1", socket=recv_sock)
send_sock.close()
recv_sock.close()
gen = spead2.send.HeapGenerator(item_group)
sender.send_heap(gen.get_heap())
sender.send_heap(gen.get_end())
received_item_group = spead2.ItemGroup()
for heap in receiver:
received_item_group.update(heap)
return received_item_group
def data_to_heaps(self, data, **kwargs):
"""Take some data and pass it through the receiver to obtain a set of heaps.
Keyword arguments are passed to the receiver constructor.
"""
thread_pool = spead2.ThreadPool()
config = recv.StreamConfig(bug_compat=self.flavour.bug_compat)
ring_config = recv.RingStreamConfig()
for key in [
'stop_on_stop_item', 'allow_unsized_heaps',
'allow_out_of_order'
]:
if key in kwargs:
setattr(config, key, kwargs.pop(key))
for key in ['contiguous_only', 'incomplete_keep_payload_ranges']:
if key in kwargs:
setattr(ring_config, key, kwargs.pop(key))
if kwargs:
raise ValueError(f'Unexpected keyword arguments ({kwargs})')
stream = recv.Stream(thread_pool, config, ring_config)
stream.add_buffer_reader(data)
return list(stream)
def transmit_item_group(self, item_group, memcpy, allocator):
if not self.spead:
raise SkipTest('spead module not importable')
thread_pool = spead2.ThreadPool(1)
sender = spead2.send.BytesStream(thread_pool)
gen = spead2.send.HeapGenerator(item_group, flavour=self.flavour)
sender.send_heap(gen.get_heap())
sender.send_heap(gen.get_end())
receiver = self.spead.TransportString(sender.getvalue())
legacy_item_group = self.spead.ItemGroup()
for heap in self.spead.iterheaps(receiver):
legacy_item_group.update(heap)
received_item_group = spead2.ItemGroup()
for key in legacy_item_group.keys():
item = legacy_item_group.get_item(key)
# PySPEAD indicates 1D variable as -1 (scalar), everything else is fixed-sized
if item.shape == -1:
shape = (None,)
else:
shape = item.shape
if item.dtype is None:
received_item_group.add_item(
id=item.id,
name=item.name,
description=item.description,
shape=shape,
format=list(self.spead.parsefmt(item.format)),
value=item.get_value())
else:
received_item_group.add_item(
id=item.id,
name=item.name,
description=item.description,
shape=shape,
dtype=item.dtype,
order='F' if item.fortran_order else 'C',
value=item.get_value())
return received_item_group
def __init__(self, spead_config):
self._config = spead_config
self._log = logging.getLogger('sip.receiver')
self._streams = []
self._item_group = spead2.ItemGroup()
self._num_buffers = self._config['num_buffers']
self._num_buffer_times = self._config['num_buffer_times']
self._num_streams = self._config['num_streams']
self._block = None
# Create the streams.
port_start = self._config['port_start']
for i_stream in range(self._num_streams):
stream = spead2.recv.asyncio.Stream(
spead2.ThreadPool(), max_heaps=1, contiguous_only=False)
pool = spead2.MemoryPool(
lower=self._config['memory_pool']['lower'],
upper=self._config['memory_pool']['upper'],
max_free=self._config['memory_pool']['max_free'],
initial=self._config['memory_pool']['initial'])
stream.set_memory_allocator(pool)
stream.add_udp_reader(port_start + i_stream)
self._streams.append(stream)
def __init__(self, spead_config):
self._config = spead_config
self._log = logging.getLogger('sip.sender')
self._streams = []
self._buffer = []
# Construct UDP streams and associated item groups.
stream_config = spead2.send.StreamConfig(
self._config['stream_config']['max_packet_size'],
self._config['stream_config']['rate'],
self._config['stream_config']['burst_size'],
self._config['stream_config']['max_heaps'])
for i_stream in range(self._config['num_streams']):
host = self._config['destination_host']
port = self._config['destination_port_start'] + i_stream
# It's much faster to have a thread pool of one thread per stream!
thread_pool = spead2.ThreadPool(threads=1)
self._log.info('Creating SPEAD stream on %s:%i ...', host, port)
udp_stream = spead2.send.asyncio.UdpStream(thread_pool, host, port,
stream_config)
item_group = spead2.send.ItemGroup(
flavour=spead2.Flavour(4, 64, 48, 0))
self._streams.append((udp_stream, item_group))
def transmit_item_groups(self,
item_groups,
*,
memcpy,
allocator,
new_order='=',
round_robin=False):
assert len(item_groups) == 1
assert not round_robin
thread_pool = spead2.ThreadPool(2)
sender = spead2.send.BytesStream(thread_pool)
gen = spead2.send.HeapGenerator(item_groups[0])
sender.send_heap(gen.get_heap())
sender.send_heap(gen.get_end())
recv_config = spead2.recv.StreamConfig(memcpy=memcpy)
if allocator is not None:
recv_config.memory_allocator = allocator
receiver = spead2.recv.Stream(thread_pool, recv_config)
receiver.add_buffer_reader(sender.getvalue())
received_item_group = spead2.ItemGroup()
for heap in receiver:
received_item_group.update(heap, new_order)
return [received_item_group]
