def test_zmq_bad_type():
def __bad_generator():
for _ in range(100):
yield dict(X=list(range(100)))
stream = pescador.Streamer(__bad_generator)
zs = pescador.ZMQStreamer(stream)
for item in zs:
pass
def test_zmq_align():
stream = pescador.Streamer(T.finite_generator, 200, size=3, lag=0.001)
reference = list(stream)
zmq_stream = pescador.ZMQStreamer(stream)
if six.PY2:
with pytest.warns(RuntimeWarning, match='array alignment'):
query = list(zmq_stream)
else:
query = list(zmq_stream)
assert len(reference) == len(query)
for b1, b2 in zip(reference, query):
T._eq_batch(b1, b2)
if six.PY2:
continue
for key in b2:
assert b2[key].flags['ALIGNED']
max_iter = 1e2 # Construct a streamer timed_sampling(data_gen, max_iter, 'Single-threaded') # Single-threaded :: Average time per iteration: 0.024 sec ############################################## # Basic ZMQ Usage ############################################## # Here is a trivial ZMQStreamer example, using it directly on top of a single # Streamer. We leave it to your imagination to decide what you would actually # do with the batches you receive here. # Wrap the streamer in a ZMQ streamer zstream = pescador.ZMQStreamer(data_gen) timed_sampling(zstream, max_iter, 'ZMQ') # ZMQ :: Average time per iteration: 0.012 sec ############################################## # ZMQ with Map Functions ############################################## # You will also likely want to buffer samples for building mini-batches. Here, # we demonstrate best practices for using map functions in a stream pipeline. buffer_size = 16 # Get batches from the stream as you would normally. batches = pescador.Streamer(pescador.buffer_stream, data_gen, buffer_size) timed_sampling(batches, max_iter, 'Single-threaded Batches') # Single-threaded Batches :: Average time per iteration: 0.392 sec