def test_buffer_stream():
inputs = [{"X": np.array([n])} for n in range(10)]
expected = [{
"X": np.array([0, 1, 2, 3]).reshape(-1, 1)
}, {
"X": np.array([4, 5, 6, 7]).reshape(-1, 1)
}, {
"X": np.array([8, 9]).reshape(-1, 1)
}]
stream = pescador.maps.buffer_stream(inputs, buffer_size=4)
outputs = list(stream)
assert len(outputs) == (len(expected) - 1)
for exp, obs in zip(expected, outputs):
T.__eq_batch(exp, obs)
stream = pescador.maps.buffer_stream(inputs, buffer_size=4, partial=True)
outputs = list(stream)
assert len(outputs) == len(expected)
for exp, obs in zip(expected, outputs):
T.__eq_batch(exp, obs)
with pytest.raises(pescador.maps.DataError):
for not_data in pescador.maps.buffer_stream([1, 2, 3, 4], 2):
pass
def test_zmq(copy, timeout):
stream = pescador.Streamer(T.finite_generator, 200, size=3, lag=0.001)
reference = list(stream)
zmq_stream = pescador.ZMQStreamer(stream, copy=copy, timeout=timeout)
for _ in range(3):
query = list(zmq_stream)
assert len(reference) == len(query)
for b1, b2 in zip(reference, query):
T.__eq_batch(b1, b2)
def test_streamer_infinite(n_max, stream_size):
reference = []
for i, data in enumerate(T.infinite_generator(size=stream_size)):
if i >= n_max:
break
reference.append(data)
streamer = pescador.core.Streamer(T.infinite_generator, size=stream_size)
for i in range(3):
query = list(streamer.iterate(max_iter=n_max))
for b1, b2 in zip(reference, query):
T.__eq_batch(b1, b2)
def test_streamer_generator_func():
n_items = 10
expected = list(T.finite_generator(n_items))
streamer = pescador.core.Streamer(T.finite_generator, n_items)
# Test generate interface
actual1 = list(streamer)
assert len(expected) == len(actual1) == n_items
for b1, b2 in zip(expected, actual1):
T.__eq_batch(b1, b2)
# Test __iter__ interface
actual2 = list(streamer)
assert len(expected) == len(actual2) == n_items
for b1, b2 in zip(expected, actual2):
T.__eq_batch(b1, b2)
def test_streamer_finite(n_max, stream_size, generate):
reference = list(T.finite_generator(50, size=stream_size))
if n_max is not None:
reference = reference[:n_max]
streamer = pescador.core.Streamer(T.finite_generator, 50, size=stream_size)
if generate:
gen = streamer.iterate(max_iter=n_max)
else:
gen = streamer(max_iter=n_max)
for i in range(3):
query = list(gen)
for b1, b2 in zip(reference, query):
T.__eq_batch(b1, b2)
def test_empty_streams():
def __empty():
if False:
yield 1
reference = pescador.Streamer(T.finite_generator, 10)
empty = pescador.Streamer(__empty)
mux = pescador.mux.Mux([reference, empty],
2,
rate=None,
with_replacement=False,
weights=[1e-10, 1e10])
estimate = list(mux.iterate(10))
ref = list(reference)
assert len(ref) == len(estimate)
for b1, b2 in zip(ref, estimate):
T.__eq_batch(b1, b2)
def test_zmq_align():
stream = pescador.Streamer(T.finite_generator, 200, size=3, lag=0.001)
reference = list(stream)
warnings.resetwarnings()
zmq_stream = pescador.ZMQStreamer(stream)
with warnings.catch_warnings(record=True) as out:
query = list(zmq_stream)
assert len(reference) == len(query)
if six.PY2:
assert len(out) > 0
assert out[0].category is RuntimeWarning
assert 'align' in str(out[0].message).lower()
for b1, b2 in zip(reference, query):
T.__eq_batch(b1, b2)
if six.PY2:
continue
for key in b2:
assert b2[key].flags['ALIGNED']
def test___stack_data():
n_items = 10
data = [{"X": np.array([n])} for n in range(n_items)]
expected = {"X": np.arange(n_items).reshape(-1, 1)}
output = pescador.maps.__stack_data(data)
T.__eq_batch(expected, output)