def test_get_star_pipe():
from liquidata import pipe, get
data = namespace_source()
a, b, f = symbolic_functions('abf')
subpipe = pipe(sym_add, f)
got = pipe(get.a.b * subpipe)(data)
assert got == [f(sym_add(n.a, n.b)) for n in data]
def test_pipe_as_component():
from liquidata import pipe, pipe
data = range(3, 6)
a, b, f, g = symbolic_functions('abfg')
a_pipe = pipe(f, g)
assert pipe(a, a_pipe,
b)(data) == list(map(b, map(g, map(f, map(a, data)))))
def test_get_star_as_args_many(where):
from liquidata import pipe, get
data = namespace_source()
if where == 'before': net = pipe(get.a.b * sym_add)
else: net = pipe(sym_add * get.a.b)
expected = list(
map(sym_add, map(attrgetter('a'), data), map(attrgetter('b'), data)))
assert net(data) == expected
def test_pick_multiple_items():
from liquidata import pipe, item as pick
names = 'abc'
ops = tuple(symbolic_functions(names))
values = range(3)
data = [{name: op(N) for (name, op) in zip(names, ops)} for N in values]
assert pipe(pick.a.b)(data) == list(map(itemgetter('a', 'b'), data))
assert pipe(pick.a)(data) == list(map(itemgetter('a'), data))
def test_final():
# ANCHOR: common_imports
import os
from keyword import iskeyword
from collections import Counter
# ANCHOR_END: common_imports
# ANCHOR: liquidata_imports
from liquidata import pipe, name as NAME, get as GET, put as PUT, join as JOIN, out as OUT, into as INTO, use
# ANCHOR_END: liquidata_imports
# ANCHOR: liquidata_full
keyword_frequency_pipe = pipe(
os.walk, JOIN, NAME.path.dirs.files,
GET.files * (JOIN, {use(str.endswith, '.py')}) >> PUT.filename,
GET.path.filename * os.path.join, open, JOIN,
use(str.split, '#', maxsplit=1), GET[0], str.split, JOIN, {iskeyword},
OUT(INTO(Counter)))
# ANCHOR_END: liquidata_full
# ANCHOR: liquidata_abstracted_full
all_files = os.walk, JOIN, NAME.path.dirs.files
pick_python_files = GET.files * (JOIN, {use(str.endswith, '.py')
}) >> PUT.filename
file_contents = GET.path.filename * os.path.join, open, JOIN
ignore_comments = use(str.split, '#', maxsplit=1), GET[0]
pick_keywords = str.split, JOIN, {iskeyword}
keyword_frequency_pipe = pipe(all_files, pick_python_files, file_contents,
ignore_comments, pick_keywords,
OUT(INTO(Counter)))
# ANCHOR_END: liquidata_abstracted_full
# ANCHOR: pure_python_full
def keyword_frequency_loop(directories):
counter = Counter()
for directory in directories:
for (path, dirs, files) in os.walk(directory):
for filename in files:
if not filename.endswith('.py'):
continue
for line in open(os.path.join(path, filename)):
for name in line.split('#', maxsplit=1)[0].split():
if iskeyword(name):
counter[name] += 1
return counter
# ANCHOR_END: pure_python_full
directories = ['/home/jacek/src']
PIPE = keyword_frequency_pipe(directories)
LOOP = keyword_frequency_loop(directories)
assert PIPE == LOOP
def test_put_operator_single_pipe(op):
from liquidata import pipe, put, get
data = namespace_source()
f, = symbolic_functions('f')
if op == ">>": net = pipe((get.b, f) >> put.f_of_b)
else: net = pipe(put.f_of_b << (get.b, f))
expected = [copy(n) for n in data]
for n in expected:
n.f_of_b = f(n.b)
assert net(data) == expected
def test_star_implicit_sink():
from liquidata import pipe, get, star, sink
data = namespace_source()
result = []
def store_sum(x, y):
result.append(sym_add(x, y))
pipe(get.a.b, sink(star(store_sum)))(data)
expected = [sym_add(ns.a, ns.b) for ns in data]
assert result == expected
def test_put_operator_single(op):
from liquidata import pipe, put
data = namespace_source()
f, = symbolic_functions('f')
def bf(ns):
return f(ns.b)
if op == ">>": net = pipe(bf >> put.f_of_b)
else: net = pipe(put.f_of_b << bf)
expected = [copy(n) for n in data]
for n in expected:
n.f_of_b = f(n.b)
assert net(data) == expected
def test_implicit_anonymous_and_named_outs():
from liquidata import pipe, out
f, g = symbolic_functions('fg')
data = range(3)
res = pipe([f, out.branch], g)(data)
assert res.branch == list(map(f, data))
assert vars(res)['return'][0] == list(map(g, data))
def test_get_single_filter():
from liquidata import pipe, get, arg as _
ds = (dict(a=1, b=2), dict(a=3, b=3), dict(a=2, b=1), dict(a=8, b=9))
data = [Namespace(**d) for d in ds]
net = pipe(get.b, {_ > 2})
expected = list(filter(_ > 2, map(attrgetter('b'), data)))
assert net(data) == expected
def test_pick_item():
from liquidata import pipe, item as pick
names = 'abc'
values = range(3)
f, = symbolic_functions('f')
data = [dict((name, value) for name in names) for value in values]
assert pipe(pick.a, f)(data) == list(map(f, values))
def test_branch():
from liquidata import pipe, sink
data = list(range(10))
branch = []
main = pipe([sink(branch.append)])(data)
assert main == data
assert branch == data
def test_put_operator_many(op):
from liquidata import pipe, put
data = namespace_source()
def sum_prod(ns):
a, b = ns.a, ns.b
return sym_add(a, b), sym_mul(a, b)
if op == ">>": net = pipe(sum_prod >> put.sum.prod)
else: net = pipe(put.sum.prod << sum_prod)
expected = [copy(n) for n in data]
for n in expected:
a, b = n.a, n.b
n.sum = sym_add(a, b)
n.prod = sym_mul(a, b)
assert net(data) == expected
def test_star_map():
from liquidata import pipe, get, star
data = namespace_source()
expected = list(
it.starmap(sym_add,
zip(map(attrgetter('a'), data), map(attrgetter('b'),
data))))
assert pipe(get.a.b, star(sym_add))(data) == expected
def test_more_than_one_implicit_anonymous_out():
from liquidata import pipe
f, g = symbolic_functions('fg')
data = range(3)
res = pipe([f], g)(data)
returns = getattr(res, 'return')
assert returns[0] == list(map(f, data))
assert returns[1] == list(map(g, data))
def test_slice_downstream(spec):
from liquidata import pipe, Slice
data = list('abcdefghij')
result = pipe(Slice(*spec))(data)
specslice = slice(*spec)
assert result == data[specslice]
assert result == data[specslice.start:specslice.stop:specslice.step]
def test_get_single_put_single():
from liquidata import pipe, get, put
data = namespace_source()
f, = symbolic_functions('f')
net = pipe((get.b, f) >> put.result)
expected = [copy(n) for n in data]
for n in expected:
n.result = f(n.b)
assert net(data) == expected
def test_integration_1():
from liquidata import pipe, arg as _, sink
data = range(20)
f, g, h = square, (_ + 1), (_ + 2)
a, b, c = odd, (_ > 50), (_ < 100)
s = []
t = pipe(f, {a}, [g, {b}, sink(s.append)], h, {c})(data)
assert s == list(filter(b, map(g, filter(a, map(f, data)))))
assert t == list(filter(c, map(h, filter(a, map(f, data)))))
def test_nested_branches():
from liquidata import pipe, out
f, g, h, i = symbolic_functions('fghi')
data = range(3)
res = pipe([[f, out.BB], g, out.BM], [h, out.MB], i, out.MM)(data)
assert res.BB == list(map(f, data))
assert res.BM == list(map(g, data))
assert res.MB == list(map(h, data))
assert res.MM == list(map(i, data))
def test_on():
from liquidata import pipe, on
names = 'abc'
f, = symbolic_functions('f')
values = range(3)
data = [Namespace(**{name: N for name in names}) for N in values]
net = pipe(on.a(f))
expected = [copy(n) for n in data]
for n in expected:
n.a = f(n.a)
assert net(data) == expected
def test_chaining_splitting_and_naming():
from liquidata import pipe, name
data = range(3)
f, g, h = symbolic_functions('fgh')
def split(x):
return f(x), g(x), h(x)
got = pipe(split, name.a.b.c)(data)
expected = list(Namespace(a=f(x), b=g(x), c=h(x)) for x in data)
assert got == expected
def test_get_single_put_many():
from liquidata import pipe, get, put
l, r = symbolic_functions('lr')
def f(x):
return l(x), r(x)
data = namespace_source()
net = pipe((get.c, f) >> put.l.r)
expected = [copy(n) for n in data]
for n in expected:
result = f(n.c)
n.l, n.r = result
assert net(data) == expected
def test_slice_close_all(close_all):
from liquidata import Slice, pipe, out
data = list(range(20))
n_elements = 5
the_slice = Slice(n_elements, close_all=close_all)
result = pipe([the_slice, out.branch], out.main)(data)
if close_all:
assert result.branch == data[:n_elements]
assert result.main == data[:n_elements]
else:
assert result.branch == data[:n_elements]
assert result.main == data
def test_fold_with_initial_value():
from liquidata import pipe, out
data = range(3)
assert pipe(out(sym_add, 99))(data) == reduce(sym_add, data, 99)
def test_fold_and_named_return():
from liquidata import pipe, out
data = range(3)
assert pipe(out.total(sym_add))(data).total == reduce(sym_add, data)
def test_while():
from liquidata import pipe, while_, arg as _
data = 'abcdXefghi'
expected = ''.join(it.takewhile(_ != 'X', data))
got = ''.join(pipe(while_(_ != 'X'))(data))
assert got == expected
def test_drop():
from liquidata import pipe, drop
data = 'abracadabra'
assert ''.join(pipe(drop(5))(data)) == ''.join(data[5:])
def test_take():
from liquidata import pipe, take
data = 'abracadabra'
assert ''.join(pipe(take(5))(data)) == ''.join(data[:5])
def test_get_many_flat():
from liquidata import pipe, flat, get
ds = (dict(a=1, b=9), dict(a=0, b=8), dict(a=3, b=7))
data = [Namespace(**d) for d in ds]
net = pipe(get.a.b * flat(lambda a, b: a * [b]))
assert net(data) == [9, 7, 7, 7]
def test_get_single_flat():
from liquidata import pipe, flat, get
ds = (dict(a=1, b=2), dict(a=0, b=3), dict(a=3, b=1))
data = [Namespace(**d) for d in ds]
net = pipe(get.a, flat(lambda n: n * [n]))
assert net(data) == [1, 3, 3, 3]