def test_push_futures_tuple():
the_source = list(range(100))
count_all = df.count()
count_odd = df.count()
result = df.push(source=the_source,
pipe=df.fork(
count_all.sink,
df.pipe(df.filter(lambda n: n % 2), count_odd.sink)),
result=(count_odd.future, count_all.future))
all_count = len(the_source)
odd_count = all_count // 2
assert result == (odd_count, all_count)
def test_push_futures_mapping():
count_all = df.count()
count_odd = df.count()
the_source = list(range(100))
result = df.push(source=the_source,
pipe=df.fork(
count_all.sink,
df.pipe(df.filter(lambda n: n % 2), count_odd.sink)),
result=dict(odd=count_odd.future, all=count_all.future))
all_count = len(the_source)
assert result.odd == all_count // 2
assert result.all == all_count
def test_push_futures_single():
the_source = list(range(100))
count = df.count()
result = df.push(source=the_source,
pipe=df.pipe(count.sink),
result=count.future)
assert result == len(the_source)
def test_push_futures():
# 'push' provides a higher-level interface to using such futures:
# it optionally accepts a tuple of futures, and returns a tuple of
# their results
count_all = df.count()
count_odd = df.count()
the_source = list(range(100))
result = df.push(source=the_source,
pipe=df.fork(
count_all.sink,
df.pipe(df.filter(lambda n: n % 2), count_odd.sink)),
result=(count_odd.future, count_all.future))
all_count = len(the_source)
odd_count = all_count // 2
assert result == (odd_count, all_count)
def test_count():
# 'count' is an example of a sink which only produces a result
# once the stream of data flowing into the pipeline has been
# closed. Such results are retrieved from futures which are
# created at the time a 'count' instance is created: a namedtuple
# containing the sink and its corresponding future is returned.
count = df.count()
the_source = list(range(30, 40))
df.push(source=the_source, pipe=count.sink)
assert count.future.result() == len(the_source)
def test_stop_when():
# 'stop_when' can be used to stop all branches of the network
# immediately.
countfuture, count = df.count()
limit, step = 10, 2
import itertools
result = df.push(source=itertools.count(start=0, step=step),
pipe=df.fork(df.stop_when(lambda n: n == limit), count),
result=(countfuture, ))
assert result == (limit // step, )
def test_stateful_stop_when():
@df.coroutine_send
def n_items_seen(n):
yield # Will stop here on construction
for _ in range(n):
yield False
yield True
countfuture, count = df.count()
import itertools
limit, step = 10, 2
result = df.push(source=itertools.count(start=0, step=step),
pipe=df.fork(df.stop_when(n_items_seen(limit)), count),
result=(countfuture, ))
assert result == (limit, )
def test_spy_count():
# count is a component that can be needed in the middle
# of a pipeline. However, because it is a sink it needs
# to be plugged into a spy. Thus, the component spy_count
# provides a comfortable interface to access the future
# and spy objects in a single line.
the_source = list(range(20))
count = df.count()
spy_count = df.spy_count()
result = df.push(source=the_source,
pipe=df.pipe(spy_count.spy, count.sink),
result=dict(from_count=count.future,
from_spy_count=spy_count.future))
assert result.from_count == result.from_spy_count == len(the_source)