def test_chain_pipes():
# Pipelines must end in sinks. If the last component of a pipe is
# not a sink, the pipe may be used as a component in a bigger
# pipeline, but it will be impossible to feed any data into it
# until it is connected to some other component which ends in a
# sink.
# Some basic pipeline components
s1 = []
sink1 = df.sink(s1.append)
s2 = []
sink2 = df.sink(s2.append)
A = df.map(lambda n: n + 1)
B = df.map(lambda n: n * 2)
C = df.map(lambda n: n - 3)
# Two different ways of creating equivalent networks: one of them
# groups the basic components into sub-pipes
graph1 = df.pipe(A, B, C, sink1)
graph2 = df.pipe(df.pipe(A, B), df.pipe(C, sink2))
# Feed the same data into the two networks
the_source = list(range(40))
df.push(source=the_source, pipe=graph1)
df.push(source=the_source, pipe=graph2)
# Confirm that both networks produce the same results.
assert s1 == s2
def test_longer_pipeline():
# Pipelines can have arbitrary lengths
the_source = list(range(1, 11))
result = []
the_sink = df.sink(result.append)
df.push(source=the_source,
pipe=df.pipe(df.map(lambda n: n + 1), df.map(lambda n: n * 2),
df.map(lambda n: n - 3), df.map(lambda n: n / 4),
the_sink))
assert result == [(((n + 1) * 2) - 3) / 4 for n in the_source]
def test_map_with_namespace_args_out():
letters = string.ascii_lowercase
the_source = (dict(i=i, x=x) for i, x in enumerate(letters))
make_upper_case = df.map(str.upper, args="x", out="upper_x")
result = []
the_sink = df.sink(result.append, args="upper_x")
df.push(source=the_source, pipe=df.pipe(make_upper_case, the_sink))
assert result == list(letters.upper())
def test_map_with_namespace_item():
# item replaces the input with the output
letters = string.ascii_lowercase
the_source = (dict(i=i, x=x) for i, x in enumerate(letters))
make_upper_case = df.map(str.upper, item="x")
result = []
the_sink = df.sink(result.append, args="x")
df.push(source=the_source, pipe=df.pipe(make_upper_case, the_sink))
assert result == list(letters.upper())
def test_branch():
# 'branch', like 'spy', allows you to insert operations on a copy
# of the stream at any point in a network. In contrast to 'spy'
# (which accepts a single plain operation), 'branch' accepts an
# arbitrary number of pipeline components, which it combines into
# a pipeline. It provides a more convenient way of constructing
# some graphs that would otherwise be constructed with 'fork'.
# Some pipeline components
c1 = []
C1 = df.sink(c1.append)
c2 = []
C2 = df.sink(c2.append)
e1 = []
E1 = df.sink(e1.append)
e2 = []
E2 = df.sink(e2.append)
A = df.map(lambda n: n + 1)
B = df.map(lambda n: n * 2)
D = df.map(lambda n: n * 3)
# Two eqivalent networks, one constructed with 'fork' the other
# with 'branch'.
graph1 = df.pipe(A, df.fork(df.pipe(B, C1), df.pipe(D, E1)))
graph2 = df.pipe(A, df.branch(B, C2), D, E2)
# Feed the same data into the two networks.
the_source = list(range(10, 50, 4))
df.push(source=the_source, pipe=graph1)
df.push(source=the_source, pipe=graph2)
# Confirm that both networks produce the same results.
assert c1 == c2
assert e1 == e2
def test_map():
# The pipelines start to become interesting when the data are
# transformed in some way. 'map' transforms every item passing
# through the pipe by applying the supplied operation.
def the_operation(n):
return n * n
square = df.map(the_operation)
the_source = list(range(1, 11))
result = []
the_sink = df.sink(result.append)
df.push(source=the_source, pipe=square(the_sink))
assert result == list(map(the_operation, the_source))
def test_fork_implicit_pipes():
# Arguments can be pipes or tuples.
# Tuples get implicitly converted into pipes
the_source = list(range(10, 20))
add_1 = df.map(lambda x: 1 + x)
implicit_pipe_collector = []
implicit_pipe_sink = df.sink(implicit_pipe_collector.append)
explicit_pipe_collector = []
explicit_pipe_sink = df.sink(explicit_pipe_collector.append)
df.push(source=the_source,
pipe=df.fork((add_1, implicit_pipe_sink),
df.pipe(add_1, explicit_pipe_sink)))
assert implicit_pipe_collector == explicit_pipe_collector == [
1 + x for x in the_source
]
def test_pipe():
# The basic syntax requires any element of a pipeline to be passed
# as argument to the one that precedes it. This looks strange to
# the human reader, especially when using parametrized
# components. 'pipe' allows construction of pipes from a sequence
# of components.
# Using 'pipe', 'test_map' could have been written like this:
def the_operation(n):
return n * n
square = df.map(the_operation)
the_source = list(range(1, 11))
result = []
the_sink = df.sink(result.append)
df.push(source=the_source, pipe=df.pipe(square, the_sink))
assert result == list(map(the_operation, the_source))
from pytest import mark
parametrize = mark.parametrize
import dataflow as df
@parametrize("component",
(df.map (lambda x: x) ,
df.filter(lambda x: x > 0),
df.sink (print) ,
df.branch(df.sink(print)) ,
df.pipe (df.map(abs)) ))
def test_string_to_pick_ignores_components(component):
assert component is df._string_to_pick(component)
def test_string_to_pick():
# string_to_pick creates a pipe component that picks
# an item from the namespace and pushes it through the pipe
the_source_elements = list(range(10))
the_source = (dict(x=i**2, y=i) for i in the_source_elements)
result = []; the_sink = df.sink(result.append)
df.push(source = the_source,
pipe = df.pipe(df._string_to_pick("y"), the_sink))
assert result == the_source_elements
def test_pipes_must_end_in_a_sink():
the_source = range(10)
sinkless_pipe = df.map(abs)
with raises(df.IncompletePipe):
df.push(source=the_source, pipe=sinkless_pipe)
def add(n):
return df.map(lambda x: x + n)