def test_take_while():
result = c.take_while(c.this < 3).as_type(list).execute(range(5))
assert result == [0, 1, 2]
result = (
c.call_func(range, c.this)
.take_while(c.this < 3)
.as_type(list)
.execute(5)
)
assert result == [0, 1, 2]
def f():
yield from range(5)
raise Exception
result = (
c.take_while(c.this < c.input_arg("stop_before"))
.filter(c.this >= c.input_arg("min_value"))
.filter(c.this < 3, cast=list)
.execute(f(), min_value=2, stop_before=4)
)
assert result == [2]
result = c.take_while(c.this < 0).as_type(list).execute(range(10))
assert result == []
def test_join_with_input_args():
assert (c.join(
c.input_arg("custom_left"),
c.input_arg("custom_right"),
c.LEFT == c.RIGHT,
).as_type(list).execute(None,
custom_left=range(3),
custom_right=range(3),
debug=True)) == [(0, 0), (1, 1), (2, 2)]
def test_pipe_label_args():
assert (c.this.pipe(
c.this,
label_input={
"label1": c.input_arg("abc")
},
label_output={
"label2": c.input_arg("cde")
},
).execute(None, abc=1, cde=2) is None)
def test_optional_dict():
conv = c.list_comp({
"key1":
c.item("key1"),
"key2":
c.optional(c.item("key2", default=None)),
"key3":
c.optional(c.item("key1") * 200, skip_value=2000),
"key4":
c.optional(
c.item("key1") * c.input_arg("x") * 300,
skip_if=c.item("key1") < 5,
),
"key5":
c.optional(
c.item("key1") * c.input_arg("x") * 300,
keep_if=c.item("key1") >= 5,
),
c.optional(c.item("key2", default=-1), skip_value=-1):
0,
c.optional(c.item("key1") * 400, skip_if=c.item("key1") < 5):
c.optional(c.item("key22")),
c.optional(c.item("key1") * 500, skip_if=c.item("key1") < 5):
c.optional(c.item("key22"), skip_value=20),
}).gen_converter(debug=False)
assert conv([{
"key1": 1,
"key2": 2
}, {
"key1": 10,
"key22": 20
}], x=1) == [
{
"key1": 1,
"key2": 2,
"key3": 200,
2: 0
},
{
"key1": 10,
"key4": 3000,
"key5": 3000,
4000: 20
},
]
with pytest.raises(Exception):
c.list_comp(c.optional(c.item("key1"))).gen_converter()
with pytest.raises(Exception):
c.optional(c.item("key1"), skip_value=1, skip_if=c.this())
with pytest.raises(Exception):
c.this().pipe(c.optional(c.this()))
class A:
x = 10
def __init__(self):
self.x = 20
conv1 = (c.this() +
c.input_arg("self").attr("x")).gen_converter(method=True)
conv2 = (c.this() +
c.input_arg("cls").attr("x")).gen_converter(method=True)
conv3 = classmethod(
(c.this() +
c.input_arg("cls").attr("x")).gen_converter(class_method=True))
conv4 = classmethod(
(c.this() +
c.input_arg("self").attr("x")).gen_converter(class_method=True))
conv5 = (c.this() + c.input_arg("self").attr("x") +
c.input_arg("n")).gen_converter(
signature="self, n=1000, data_=15")
conv6 = staticmethod(
((c.this() + c.call_func(sum, c.input_arg("args"))) *
c.input_arg("kwargs").call_method("get", "multiplicator", 1)
).gen_converter(signature="data_, *args, **kwargs"))
def test_labels():
conv1 = c.if_(
1,
c.input_arg("y").item("abc").add_label("abc").pipe(
c.input_arg("x").pipe(
c.inline_expr("{cde} + 10").pass_args(
cde=c.this().item("cde")))).pipe(
c.inline_expr("{this} + {abc}").pass_args(
this=c.this(), abc=c.label("abc"))),
2,
).gen_converter(debug=False)
assert conv1(data_=1, x={"cde": 2}, y={"abc": 3}) == 15
list(c.generator_comp(c.this().add_label("a")).execute([1, 2]))
c.list_comp(c.this().add_label("a")).execute([1, 2])
def test_pipes():
assert c.list_comp(c.inline_expr("{0} ** 2").pass_args(c.this())).pipe(
c.call_func(sum, c.this())).pipe(
c.call_func(
lambda x, a: x + a,
c.this(),
c.naive({
"abc": 10
}).item(c.input_arg("key_name")),
)).pipe([c.this(), c.this()]).execute([1, 2, 3],
key_name="abc",
debug=False) == [
24,
24,
]
assert c.item(0).pipe(
datetime.strptime,
"%Y-%m-%d",
).pipe(c.call_func(lambda dt: dt.date(),
c.this())).execute([
"2019-01-01",
], debug=False) == date(2019, 1, 1)
assert c.item(0).pipe(
datetime.strptime,
"%Y-%m-%d",
).pipe(c.this().call_method("date")).execute([
"2019-01-01",
], debug=False) == date(2019, 1, 1)
with pytest.raises(c.ConversionException):
c.naive(True).pipe(c.item("key1", _predefined_input={"key1": 777}))
def test_input_arg():
assert c.input_arg("x").as_type(int).execute(None, x="10") == 10
assert (
c.inline_expr(""""{{}}_{{}}".format(type({x}).__name__, {x})""")
.pass_args(x=c.item("value"))
.gen_converter()
)({"value": 123}) == "int_123"
def test_simple_label():
conv1 = (c.tuple(c.item(2).add_label("a"), c.this()).pipe(
c.item(1).pipe(c.list_comp(
(c.this(), c.label("a"))))).gen_converter(debug=False))
assert conv1([1, 2, 3, 4]) == [(1, 3), (2, 3), (3, 3), (4, 3)]
conv2 = (c.tuple(c.item(1).add_label("a"), c.this()).pipe(
c.item(1),
label_input={
"aa": c.item(0),
"bb": c.item(0)
},
label_output="collection1",
).pipe(
c.label("collection1").pipe(
c.aggregate(
c.ReduceFuncs.Sum(
c.this() + c.label("a") + c.label("aa") +
c.input_arg("x") + c.label("collection1").item(0), ))),
label_output="b",
).pipe(c.this() + c.label("b")).gen_converter(debug=False))
assert conv2([1, 2, 3, 4], x=10) == 140
conv3 = (c.tuple(c.item("default").add_label("default"), c.this()).pipe(
c.item(1).pipe(c.item(
"abc", default=c.label("default")))).gen_converter(debug=False))
assert conv3({"default": 1}) == 1
with pytest.raises(c.ConversionException):
c.this().pipe(c.this(), label_input=1)
def test_slices():
assert c.this()[c.item(0):c.input_arg("slice_to"):c.item(1)].gen_converter(
debug=False)([2, 2, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], slice_to=8) == [
1,
3,
5,
]
def test_slices():
assert c.this[c.item(0):c.input_arg("slice_to"):c.item(1)].execute(
[2, 2, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], slice_to=8) == [
1,
3,
5,
]
def test_group_by_with_double_ended_pipes():
input_data = [
{
"value": 1
},
{
"value": 2
},
]
# fmt: off
conv = c.aggregate(
c.item("value").pipe(c.ReduceFuncs.Sum(c.this())).pipe(
c.this() * 2)).gen_converter()
# fmt: on
result = conv(input_data)
assert result == 6
input_data = [
{
"k": "A",
"v": 1
},
{
"k": "A",
"v": 2
},
]
reducer = c.ReduceFuncs.Sum(c.item("v"))
conv = (c.group_by(c.item("k")).aggregate({
"v1":
c.input_arg("test").pipe(reducer),
"v2":
reducer,
}).gen_converter())
assert conv(input_data, test={"v": 7}) == [{"v1": 14, "v2": 3}]
def test_bad_namespace_usage():
with pytest.raises(Exception,
match="rendering prevented by parent NamespaceCtx"):
assert BadDropWhile(c.this < 100).gen_converter()
with pytest.raises(Exception,
match="rendering prevented by parent NamespaceCtx"):
assert BadDropWhile(c.this < c.input_arg("abc")).gen_converter()
def test_nested_loop_joins():
from convtools.conversion import join
join1 = (
join(
c.item(0),
c.item(1),
c.and_(
c.LEFT.item("id") - 1 == c.RIGHT.item("ID"),
c.LEFT.item("value") + 100 < c.RIGHT.item("value"),
c.LEFT.item("value") > 101,
c.RIGHT.item("value") < 209,
c.input_arg("flag"),
),
)
.as_type(list)
.gen_converter(debug=True)
)
assert join1(
[
[{"id": i, "value": i + 100} for i in range(10)],
[{"ID": i, "value": 210 - i} for i in range(10)],
],
flag=True,
) == [
({"id": 3, "value": 103}, {"ID": 2, "value": 208}),
({"id": 4, "value": 104}, {"ID": 3, "value": 207}),
({"id": 5, "value": 105}, {"ID": 4, "value": 206}),
]
join2 = (
join(
c.item(0),
c.item(1),
c.and_(c.LEFT * c.RIGHT < 30, c.LEFT + c.RIGHT > 8),
)
.as_type(list)
.gen_converter(debug=False)
)
assert join2([range(4, 10), range(4, 10)]) == [
(4, 5),
(4, 6),
(4, 7),
(5, 4),
(5, 5),
(6, 4),
(7, 4),
]
join3 = (
join(c.item(0), c.item(1), Eq(c.LEFT + c.RIGHT, 1))
.as_type(list)
.gen_converter(debug=True)
)
assert join3(([-1, 0, 1], [2, 1, 1])) == [(-1, 2), (0, 1), (0, 1)]
def test_manually_defined_reducers():
data = [
{
"name": "John",
"category": "Games",
"debit": 10,
"balance": 90
},
{
"name": "John",
"category": "Games",
"debit": 200,
"balance": -110
},
{
"name": "John",
"category": "Food",
"debit": 30,
"balance": -140
},
{
"name": "John",
"category": "Games",
"debit": 300,
"balance": 0
},
{
"name": "Nick",
"category": "Food",
"debit": 7,
"balance": 50
},
{
"name": "Nick",
"category": "Games",
"debit": 18,
"balance": 32
},
{
"name": "Bill",
"category": "Games",
"debit": 18,
"balance": 120
},
]
grouper = (c.group_by(c.item("name")).aggregate(
c.reduce(lambda a, b: a + b,
c.item(c.input_arg("group_key")),
initial=0)).filter(c.this() > 20).gen_converter(
signature="data_, group_key='debit'"))
assert grouper(data) == [540, 25]
assert grouper(data, group_key="balance") == [82, 120]
def test_pipes():
assert c.list_comp(c.inline_expr("{0} ** 2").pass_args(c.this)).pipe(
c.call_func(sum, c.this)).pipe(
c.call_func(
lambda x, a: x + a,
c.this,
c.naive({
"abc": 10
}).item(c.input_arg("key_name")),
)).pipe([c.this, c.this]).execute([1, 2, 3],
key_name="abc",
debug=False) == [
24,
24,
]
assert c.item(0).pipe(datetime.strptime, "%Y-%m-%d").pipe(
c.call_func(lambda dt: dt.date(),
c.this)).execute(["2019-01-01"],
debug=False) == date(2019, 1, 1)
assert c.item(0).pipe(datetime.strptime, "%Y-%m-%d").pipe(
c.this.call_method("date")).execute(["2019-01-01"],
debug=False) == date(2019, 1, 1)
conv = c.dict_comp(
c.item("name"),
c.item("transactions").pipe(
c.list_comp({
"id":
c.item(0).as_type(str),
"amount":
c.item(1).pipe(c.if_(c.this, c.this.as_type(Decimal), None)),
})),
).gen_converter(debug=False)
assert conv([{
"name": "test",
"transactions": [(0, 0), (1, 10)]
}]) == {
"test": [
{
"id": "0",
"amount": None
},
{
"id": "1",
"amount": Decimal("10")
},
]
}
assert c.this.pipe(lambda it: it).filter(
c.this).sort().as_type(list).execute((2, 1, 0)) == [1, 2]
def test_cloning_functionality():
conv1 = c.item("a")
conv2 = c.item("c").pipe(c.item("b").pipe(conv1))
conv3 = c.item("d").pipe(c.item(c.input_arg("test")).pipe(conv1))
assert conv3.execute({"d": {"X": {"a": 2}}}, test="X") == 2
assert conv2.execute({"c": {"b": {"a": 1}}}) == 1
conv1 = c.item("a")
conv2 = conv1.item("b")
conv3 = conv1.item("c")
assert conv3.execute({"a": {"c": 3}}) == 3
assert conv2.execute({"a": {"b": 2}}) == 2
def test_drop_while():
result = c.drop_while(c.this < 3).as_type(list).execute(range(5))
assert result == [3, 4]
result = (
c.call_func(range, c.this)
.drop_while(c.this < c.input_arg("min_value"))
.as_type(list)
.execute(5, min_value=3)
)
assert result == [3, 4]
result = c.drop_while(c.this >= 0).as_type(list).execute(range(10))
assert result == []
def test_mutation_item():
now = datetime.now()
assert c.list_comp(
{
"name": c.item("fullName"),
"age": c.item("age").as_type(int),
"to_del": 1,
}
).pipe(
c.list_comp(
c.call_func(lambda d: d, c.this).tap(
c.Mut.set_item(
"name_before", c.label("_input").item(0, "name")
),
c.Mut.set_item("name", c.item("name").call_method("lower")),
c.Mut.set_item(
"name_after", c.label("_input").item(0, "name")
),
c.Mut.set_item("_updated", c.input_arg("now")),
c.Mut.set_item(c.item("age"), c.item("age") >= 18),
c.Mut.del_item("to_del"),
c.Mut.custom(c.this.call_method("update", {"to_add": 2})),
c.this.call_method("update", {"to_add2": 4}),
)
),
label_input="_input",
).execute(
[{"fullName": "John", "age": "28"}], debug=False, now=now
) == [
{
"name": "john",
"name_after": "john",
"name_before": "John",
"age": 28,
"_updated": now,
28: True,
"to_add": 2,
"to_add2": 4,
}
]
with pytest.raises(Exception):
c.item(c.Mut.set_item("abc", "cde"))
with pytest.raises(Exception):
conversion = c.item(1)
conversion.ensure_conversion(
c.Mut.set_item("abc", "cde"), explicitly_allowed_cls=GetItem
)
def test_is_independent():
assert c(0).is_independent()
assert c(int).is_independent()
assert c(int).call().is_independent()
assert c.label("a").is_independent()
assert c.inline_expr("{}()").pass_args(int).is_independent()
assert c.escaped_string("int()").is_independent()
assert c({"a": c.input_arg("key")}).is_independent()
assert not c.iter({"a": 1}).is_independent()
assert not c.this.is_independent()
assert not c({"a": 1}).item("a").is_independent()
assert not c({"a": 1}).item(c.item("a")).is_independent()
assert not c.inline_expr("{}()").pass_args(c.this).is_independent()
assert not c.aggregate({"a": 1}).is_independent()
assert not c.this.add_label("a").is_independent()
assert not c(int).call(c.item(0)).is_independent()
def test_namespaces():
with pytest.raises(ValueError):
LazyEscapedString("abc").execute([1])
with pytest.raises(ValueError):
Namespace(LazyEscapedString("abc"), name_to_code={
"abc": None
}).execute([1])
assert (Namespace(LazyEscapedString("abc"), name_to_code={
"abc": True
}).execute(1) == 1)
assert (Namespace(
c.input_arg("abc") + LazyEscapedString("abc"),
name_to_code={
"abc": "abc"
},
).execute(0.1, abc=2) == 4)
assert Namespace(c.item(1), {}).execute([0, 10]) == 10
assert (Namespace(
Namespace(
Namespace(LazyEscapedString("abc"), name_to_code={"abc": True
}) # 1
+ LazyEscapedString("abc") # 10
+ LazyEscapedString("foo") # 1000
+ c.item() * 0.1, # 0.1,
name_to_code={"foo": "arg_foo2"},
),
name_to_code={
"abc": "arg_abc",
"foo": "arg_foo"
},
)).gen_converter(
debug=False,
signature="data_, arg_abc=10, arg_foo=100, arg_foo2=1000")(1) == 1011.1
assert (Namespace(
c.call_func(list, (1, )).pipe(
c.if_(
c.this,
c.this * LazyEscapedString("number"),
c.this,
)),
{
"number": "3"
},
).execute(None) == [1, 1, 1])
def test_iter_mut_method():
assert c.iter(c.item(0)).as_type(list).execute([[1], [2]]) == [1, 2]
assert c.iter_mut(c.Mut.custom(c.this.call_method("append", 7))).as_type(
list
).execute([[1], [2]]) == [[1, 7], [2, 7]]
result = (
c.this.iter({"a": c.this})
.iter_mut(
c.Mut.set_item("b", c.item("a") + 1),
c.Mut.set_item("c", c.item("a") + 2),
)
.iter_mut(
c.Mut.set_item("d", c.item("a") + 3),
)
.as_type(list)
.execute([1, 2, 3], debug=False)
)
assert result == [
{"a": 1, "b": 2, "c": 3, "d": 4},
{"a": 2, "b": 3, "c": 4, "d": 5},
{"a": 3, "b": 4, "c": 5, "d": 6},
]
result = (
c.group_by(c.item(0))
.aggregate(
c(
[
{c.item(0): c.item(1).pipe(c.ReduceFuncs.Max(c.this))},
{c.item(1).pipe(c.ReduceFuncs.Max(c.this)): c.item(0)},
]
)
.iter_mut(
c.Mut.set_item(
"x",
c.call_func(sum, c.this.call_method("values"))
+ c.input_arg("base"),
)
)
.as_type(tuple)
)
.execute([(0, 1), (0, 2), (1, 7)], base=100, debug=False)
)
assert result == [
({0: 2, "x": 102}, {2: 0, "x": 100}),
({1: 7, "x": 107}, {7: 1, "x": 101}),
]
def test_hashes():
assert hash(c.input_arg("abc")) == hash(c.input_arg("abc"))
assert hash(c.input_arg("abd")) != hash(c.input_arg("abc"))
assert hash(c.inline_expr("abc")) == hash(c.inline_expr("abc"))
assert hash(c.inline_expr("abd")) != hash(c.inline_expr("abc"))
def test_join_conditions():
join_conditions = _JoinConditions.from_condition(c.LEFT == c.RIGHT)
assert (True and join_conditions.inner_loop_conditions == []
and join_conditions.left_collection_filters == []
and join_conditions.left_row_filters == []
and join_conditions.left_row_hashers == [c.LEFT]
and join_conditions.pre_filter == []
and join_conditions.right_collection_filters == []
and join_conditions.right_row_filters == []
and join_conditions.right_row_hashers == [c.RIGHT])
join_conditions = _JoinConditions.from_condition(
c.or_(c.LEFT == c.RIGHT, c.LEFT == c.RIGHT))
c11 = c.LEFT.item(0)
c12 = c.RIGHT.item(1)
c21 = c.LEFT.item(1)
c22 = c.RIGHT.item(0)
c13 = c.LEFT.item(2) > 10
c23 = c.RIGHT.item(2) < 10
c01 = c.input_arg("x") > 100
join_conditions = _JoinConditions.from_condition(
c.and_(c11 == c12, c22 == c21, c13).and_(c23, c01))
assert (True and join_conditions.inner_loop_conditions == []
and join_conditions.left_collection_filters == [c13]
and join_conditions.left_row_filters == []
and join_conditions.left_row_hashers == [c11, c21]
and join_conditions.pre_filter == [c01]
and join_conditions.right_collection_filters == [c23]
and join_conditions.right_row_filters == []
and join_conditions.right_row_hashers == [c12, c22])
join_conditions = _JoinConditions.from_condition(c.and_(
c11 == c12, c22 == c21, c13).and_(c23, c01),
how="left")
assert (True and join_conditions.inner_loop_conditions == []
and join_conditions.left_collection_filters == []
and join_conditions.left_row_filters == [c13]
and join_conditions.left_row_hashers == [c11, c21]
and join_conditions.pre_filter == [c01]
and join_conditions.right_collection_filters == [c23]
and join_conditions.right_row_filters == []
and join_conditions.right_row_hashers == [c12, c22])
join_conditions = _JoinConditions.from_condition(c.and_(
c11 == c12, c22 == c21, c13).and_(c23, c01),
how="right")
assert (True and join_conditions.inner_loop_conditions == []
and join_conditions.left_collection_filters == [c13]
and join_conditions.left_row_filters == []
and join_conditions.left_row_hashers == [c11, c21]
and join_conditions.pre_filter == [c01]
and join_conditions.right_collection_filters == []
and join_conditions.right_row_filters == [c23]
and join_conditions.right_row_hashers == [c12, c22])
join_conditions = _JoinConditions.from_condition(c.and_(
c11 == c12, c22 == c21, c13).and_(c23, c01),
how="outer")
assert (True and join_conditions.inner_loop_conditions == []
and join_conditions.left_collection_filters == []
and join_conditions.left_row_filters == [c13]
and join_conditions.left_row_hashers == [c11, c21]
and join_conditions.pre_filter == [c01]
and join_conditions.right_collection_filters == []
and join_conditions.right_row_filters == [c23]
and join_conditions.right_row_hashers == [c12, c22])
with pytest.raises(AssertionError):
_JoinConditions.from_condition(c.and_(True, False), how="abc")
c1 = c.LEFT != c.RIGHT
join_conditions = _JoinConditions.from_condition(c1)
assert (True and join_conditions.inner_loop_conditions == [c1]
and join_conditions.left_collection_filters == []
and join_conditions.left_row_filters == []
and join_conditions.left_row_hashers == []
and join_conditions.pre_filter == []
and join_conditions.right_collection_filters == []
and join_conditions.right_row_filters == []
and join_conditions.right_row_hashers == [])
cond = c.LEFT > c.RIGHT
join_conditions = _JoinConditions.from_condition(cond)
assert (True and join_conditions.inner_loop_conditions == [cond]
and join_conditions.left_collection_filters == []
and join_conditions.left_row_filters == []
and join_conditions.left_row_hashers == []
and join_conditions.pre_filter == []
and join_conditions.right_collection_filters == []
and join_conditions.right_row_filters == []
and join_conditions.right_row_hashers == [])
c1 = c.LEFT == 1
c2 = c.RIGHT == 1
c3 = c.input_arg("x") == 1
join_conditions = _JoinConditions.from_condition(c1.and_(c2, c3),
how="outer")
assert (True and join_conditions.inner_loop_conditions == []
and join_conditions.left_collection_filters == []
and join_conditions.left_row_filters == [c1]
and join_conditions.left_row_hashers == []
and join_conditions.pre_filter == [c3]
and join_conditions.right_collection_filters == []
and join_conditions.right_row_filters == [c2]
and join_conditions.right_row_hashers == [])
c1 = c.LEFT + c.RIGHT + 10
c2 = c.LEFT + 1
c3 = c.RIGHT + 1
join_conditions = _JoinConditions.from_condition(c.and_(c1, c2, c3))
assert (True and join_conditions.inner_loop_conditions == [c1]
and join_conditions.left_collection_filters == [c2]
and join_conditions.left_row_filters == []
and join_conditions.left_row_hashers == []
and join_conditions.pre_filter == []
and join_conditions.right_collection_filters == [c3]
and join_conditions.right_row_filters == []
and join_conditions.right_row_hashers == [])
def test_hash_joins():
join1 = (c.join(
c.item(0), c.item(1),
c.LEFT.item("id") == c.RIGHT.item("id")).as_type(list).gen_converter(
debug=False))
join1([
[{
"id": i,
"value": i + 100
} for i in range(3)],
[{
"id": i,
"value": i + 200
} for i in range(3)],
]) == [
({
"id": 0,
"value": 100
}, {
"id": 0,
"value": 200
}),
({
"id": 1,
"value": 101
}, {
"id": 1,
"value": 201
}),
({
"id": 2,
"value": 102
}, {
"id": 2,
"value": 202
}),
]
join2 = (c.join(
c.item(0),
c.item(1),
c.and_(
c.LEFT.item("id") == c.RIGHT.item("ID"),
c.LEFT.item("value") > 105,
c.RIGHT.item("value") < 209,
c.input_arg("flag"),
),
).as_type(list).gen_converter(debug=False))
assert join2(
[
[{
"id": i,
"value": i + 100
} for i in range(10)],
[{
"ID": i,
"value": i + 200
} for i in range(10)],
],
flag=True,
) == [
({
"id": 6,
"value": 106
}, {
"ID": 6,
"value": 206
}),
({
"id": 7,
"value": 107
}, {
"ID": 7,
"value": 207
}),
({
"id": 8,
"value": 108
}, {
"ID": 8,
"value": 208
}),
]
assert (join2(
[
[{
"id": i,
"value": i + 100
} for i in range(10)],
[{
"ID": i,
"value": i + 200
} for i in range(10)],
],
flag=False,
) == [])
join2 = (c.join(
c.item(0),
c.item(1),
c.and_(
c.LEFT.item("id") == c.RIGHT.item("ID"),
c.LEFT.item("value") > 105,
c.RIGHT.item("value") < 209,
c.input_arg("flag"),
),
).as_type(list).gen_converter(debug=False))
assert join2(
[
[{
"id": i,
"value": i + 100
} for i in range(10)],
[{
"ID": i,
"value": i + 200
} for i in range(10)],
],
flag=True,
) == [
({
"id": 6,
"value": 106
}, {
"ID": 6,
"value": 206
}),
({
"id": 7,
"value": 107
}, {
"ID": 7,
"value": 207
}),
({
"id": 8,
"value": 108
}, {
"ID": 8,
"value": 208
}),
]
def test_grouping():
data = [
{
"name": "John",
"category": "Games",
"debit": 10,
"balance": 90
},
{
"name": "John",
"category": "Games",
"debit": 200,
"balance": -110
},
{
"name": "John",
"category": "Food",
"debit": 30,
"balance": -140
},
{
"name": "John",
"category": "Games",
"debit": 300,
"balance": 0
},
{
"name": "Nick",
"category": "Food",
"debit": 7,
"balance": 50
},
{
"name": "Nick",
"category": "Games",
"debit": 18,
"balance": 32
},
{
"name": "Bill",
"category": "Games",
"debit": 18,
"balance": 120
},
]
result = (c.group_by(c.item("name")).aggregate((
c.item("name"),
c.item("name").call_method("lower"),
c.call_func(str.lower, c.item("name")),
c.reduce(
lambda a, b: a + b,
c.item("debit"),
initial=c.input_arg("arg1"),
unconditional_init=True,
),
c.reduce(
c.inline_expr("{0} + {1}"),
c.item("debit"),
initial=lambda: 100,
unconditional_init=True,
),
c.reduce(
max,
c.item("debit"),
prepare_first=lambda a: a,
default=c.input_arg("arg1"),
where=c.call_func(lambda x: x < 0, c.item("balance")),
),
c.call_func(
lambda max_debit, n: max_debit * n,
c.reduce(
max,
c.item("debit"),
prepare_first=lambda a: a,
default=0,
where=c.call_func(lambda x: x < 0, c.item("balance")),
),
1000,
),
c.call_func(
lambda max_debit, n: max_debit * n,
c.reduce(
c.ReduceFuncs.Max,
c.item("debit"),
default=1000,
where=c.inline_expr("{0} > {1}").pass_args(
c.item("balance"),
c.input_arg("arg2"),
),
),
-1,
),
c.reduce(c.ReduceFuncs.MaxRow, c.item("debit")).item("balance"),
c.reduce(c.ReduceFuncs.MinRow, c.item("debit")).item("balance"),
)).sort(key=lambda t: t[0].lower(), reverse=True).execute(data,
arg1=100,
arg2=0,
debug=False))
# fmt: off
assert result == [
('Nick', 'nick', 'nick', 125, 125, 100, 0, -18, 32, 50),
('John', 'john', 'john', 640, 640, 200, 200000, -10, 0, 90),
('Bill', 'bill', 'bill', 118, 118, 100, 0, -18, 120, 120),
]
# fmt: on
with pytest.raises(c.ConversionException):
# there's a single group by field, while we use separate items
# of this tuple in aggregate
result = (c.group_by(c.item("name")).aggregate((
c.item("category"),
c.reduce(c.ReduceFuncs.Sum, c.item("debit")),
)).execute(data, debug=False))
aggregation = {
c.call_func(
tuple,
c.ReduceFuncs.Array(c.item("name"), default=None),
):
c.item("category").call_method("lower"),
"count":
c.ReduceFuncs.Count(),
"max":
c.ReduceFuncs.Max(c.item("debit")),
"min":
c.ReduceFuncs.Min(c.item("debit")),
"count_distinct":
c.ReduceFuncs.CountDistinct(c.item("name")),
"array_agg_distinct":
c.ReduceFuncs.ArrayDistinct(c.item("name")),
"dict":
c.ReduceFuncs.Dict(c.item("debit"), c.item("name")),
}
result = (c.group_by(c.item("category")).aggregate(aggregation).execute(
data, debug=False))
result2 = (c.group_by(c.item("category")).aggregate(
c.dict(*aggregation.items())).execute(data, debug=False))
# fmt: off
assert result == result2 == [
{
'array_agg_distinct': ['John', 'Nick', 'Bill'],
'count': 5,
'count_distinct': 3,
'dict': {
10: 'John',
18: 'Bill',
200: 'John',
300: 'John'
},
'max': 300,
'min': 10,
('John', 'John', 'John', 'Nick', 'Bill'): 'games'
}, {
'array_agg_distinct': ['John', 'Nick'],
'count': 2,
'count_distinct': 2,
'dict': {
7: 'Nick',
30: 'John'
},
'max': 30,
'min': 7,
('John', 'Nick'): 'food'
}
]
# fmt: on
result3 = (c.aggregate(c.ReduceFuncs.Sum(c.item("debit"))).pipe(
c.inline_expr("{0} + {1}").pass_args(c.this(),
c.this())).execute(data,
debug=False))
assert result3 == 583 * 2
by = c.item("name"), c.item("category")
result4 = (c.group_by(
*by).aggregate(by + (c.ReduceFuncs.Sum(c.item("debit")), )).execute(
data, debug=False))
# fmt: off
assert result4 == [('John', 'Games', 510), ('John', 'Food', 30),
('Nick', 'Food', 7), ('Nick', 'Games', 18),
('Bill', 'Games', 18)]
# fmt: on
result5 = (c.group_by().aggregate(c.ReduceFuncs.Sum(
c.item("debit"))).execute(data, debug=False))
assert result5 == 583
with pytest.raises(c.ConversionException):
# there's a single group by field, while we use separate items
# of this tuple in aggregate
(c.group_by(by).aggregate(
by + (c.reduce(c.ReduceFuncs.Sum, c.item("debit")), )).execute(
data, debug=False))
def test_gen_converter():
class A:
x = 10
def __init__(self):
self.x = 20
conv1 = (c.this() +
c.input_arg("self").attr("x")).gen_converter(method=True)
conv2 = (c.this() +
c.input_arg("cls").attr("x")).gen_converter(method=True)
conv3 = classmethod(
(c.this() +
c.input_arg("cls").attr("x")).gen_converter(class_method=True))
conv4 = classmethod(
(c.this() +
c.input_arg("self").attr("x")).gen_converter(class_method=True))
conv5 = (c.this() + c.input_arg("self").attr("x") +
c.input_arg("n")).gen_converter(
signature="self, n=1000, data_=15")
conv6 = staticmethod(
((c.this() + c.call_func(sum, c.input_arg("args"))) *
c.input_arg("kwargs").call_method("get", "multiplicator", 1)
).gen_converter(signature="data_, *args, **kwargs"))
assert A().conv1(100) == 120
assert A.conv3(100) == 110
with pytest.raises(NameError):
A().conv2(100)
with pytest.raises(NameError):
A.conv4(100)
assert A().conv5() == 1035
assert A().conv5(data_=7) == 1027
assert A().conv5(n=100) == 135
assert A.conv6(20) == 20
assert A.conv6(20, 1, 2, 3) == 26
assert A.conv6(20, 1, 2, 3, multiplicator=10) == 260
assert (c.call_func(sum,
c.this()).gen_converter(signature="*data_")(1, 2,
3) == 6)
assert (c.call_func(lambda i: globals().__setitem__("A", 1) or sum(i),
c.this()).gen_converter(signature="*data_")(1, 2,
3) == 6)
assert c({
c.naive("-").call_method("join",
c.this().call_method("keys")):
c.call_func(sum,
c.this().call_method("values"))
}).gen_converter(signature="**data_")(a=1, b=2, c=3) == {
"a-b-c": 6
}
with pytest.raises(c.ConversionException):
c.call_func(sum,
c.input_arg("x")).gen_converter(signature="*data_")(1, 2,
3)
with pytest.raises(c.ConversionException):
c.this().gen_converter(method=True, class_method=True)
def test_conversions_dependencies():
input_arg = c.input_arg("abc")
conv = c.item(input_arg)
assert tuple(conv.get_dependencies()) == (input_arg, conv)
def test_doc__index_word_count():
# Let's say we need to count words across all files
input_data = [
"war-and-peace-1.txt",
"war-and-peace-2.txt",
"war-and-peace-3.txt",
"war-and-peace-4.txt",
]
# # iterate an input and read file lines
#
# def read_file(filename):
# with open(filename) as f:
# for line in f:
# yield line
# extract_strings = c.generator_comp(c.call_func(read_file, c.this()))
# to simplify testing
extract_strings = c.generator_comp(
c.call_func(lambda filename: [filename], c.this()))
# 1. make ``re`` pattern available to the code to be generated
# 2. call ``finditer`` method of the pattern and pass the string
# as an argument
# 3. pass the result to the next conversion
# 4. iterate results, call ``.group()`` method of each re.Match
# and call ``.lower()`` on each result
split_words = (c.naive(re.compile(r"\w+")).call_method(
"finditer", c.this()).pipe(
c.generator_comp(c.this().call_method("group",
0).call_method("lower"))))
# ``extract_strings`` is the generator of strings
# so we iterate it and pass each item to ``split_words`` conversion
vectorized_split_words = c.generator_comp(c.this().pipe(split_words))
# flattening the result of ``vectorized_split_words``, which is
# a generator of generators of strings
flatten = c.call_func(
chain.from_iterable,
c.this(),
)
# aggregate the input, the result is a single dict
# words are keys, values are count of words
dict_word_to_count = c.aggregate(
c.ReduceFuncs.DictCount(c.this(), c.this(), default=dict))
# take top N words by:
# - call ``.items()`` method of the dict (the result of the aggregate)
# - pass the result to ``sorted``
# - take the slice, using input argument named ``top_n``
# - cast to a dict
take_top_n = (c.this().call_method("items").sort(
key=lambda t: t[1],
reverse=True).pipe(c.this()[:c.input_arg("top_n")]).as_type(dict))
# the resulting pipeline is pretty self-descriptive, except the ``c.if_``
# part, which checks the condition (first argument),
# and returns the 2nd if True OR the 3rd (input data by default) otherwise
pipeline = (
extract_strings.pipe(flatten).pipe(vectorized_split_words).pipe(
flatten).pipe(dict_word_to_count).pipe(
c.if_(
c.input_arg("top_n").is_not(None),
c.this().pipe(take_top_n),
))
# Define the resulting converter function signature. In fact this
# isn't necessary if you don't need to specify default values
).gen_converter(debug=True, signature="data_, top_n=None")
assert pipeline(input_data, top_n=3) == {"war": 4, "and": 4, "peace": 4}
def test_naive_conversion_item():
d = {1: 2, 10: {"test": 15, 2: 777}, 100: {"test2": 200}}
assert c.naive(d).item(1).execute(100) == 2
assert c.item(1).gen_converter()(d) == 2
assert c.item(10, "test").gen_converter()(d) == 15
assert c.item(11, "test", default=77).gen_converter()(d) == 77
assert (c.item(10, c.input_arg("arg1"),
default=c.input_arg("arg2")).gen_converter()(d,
arg1="test",
arg2=77) == 15)
assert (c.item(10, c.input_arg("arg1"),
default=c.input_arg("arg2")).gen_converter()(d,
arg1="tst",
arg2=77) == 77)
assert (c.item(11, "test", default=77).gen_converter(method=True)(None,
d) == 77)
assert c.item(10, "testt", default=77).gen_converter()(d) == 77
assert c.item(10, "testt", default=c.this()).gen_converter()(d) == d
assert c.item(10, c.item(1)).gen_converter()(d) == 777
assert c.item(10).item(2).gen_converter()(d) == 777
with pytest.raises(KeyError):
c.naive(d).item(11).gen_converter()(100)
with pytest.raises(IndexError):
c.naive([]).item(11).gen_converter()(100)
with pytest.raises(TypeError):
c.naive(None).item(11).gen_converter()(100)
assert (c.naive(d).item(100).item("test2").gen_converter(
debug=False)(100) == 200)
assert (c.naive(d).item(c.this(),
"test2").gen_converter(debug=False)(100) == 200)
assert (c.naive(d).item(100, default=30).item(
"test2", default=30).gen_converter(debug=False)(100) == 200)
# testing defaults
assert (c.naive(d).item(100, default=30).item(
"test", default=30).gen_converter()(100) == 30)
assert (c.naive(d).item(10).item("test2",
default=30).gen_converter()(100) == 30)
assert c.naive(True).is_(True).execute(100) is True
assert c.naive(True).is_not(True).execute(100) is False
assert c.naive(1).in_({1, 2}).execute(100) is True
assert c.naive(1).in_({3, 2}).execute(100) is False
assert c.naive(1).not_in({3, 2}).execute(100) is True
assert c.naive(1).eq(1).execute(100) is True
assert (c.naive(1) == 1).execute(100) is True
assert c.naive(1).not_eq(1).execute(100) is False
assert (c.naive(1) != 1).execute(100) is False
assert c.naive(1).gte(1).execute(100) is True
assert (c.naive(1) >= 1).execute(100) is True
assert c.naive(2).gte(1).execute(100) is True
assert (c.naive(2) >= 1).execute(100) is True
assert c.naive(10).gt(1).execute(100) is True
assert (c.naive(10) > 1).execute(100) is True
assert c.naive(1).lte(1).execute(100) is True
assert (c.naive(1) <= 1).execute(100) is True
assert c.naive(0).lte(1).execute(100) is True
assert (c.naive(0) <= 1).execute(100) is True
assert c.naive(0).lt(1).execute(100) is True
assert (c.naive(0) < 1).execute(100) is True
assert c.this().neg().execute(2) == -2
assert (-c.this()).execute(2) == -2
assert (c.this() + c.this()).execute(2) == 4
assert (c.this() * c.this()).execute(3) == 9
assert (c.this() - c.this()).execute(2) == 0
assert (c.naive(5) / c.this()).execute(2) == 2.5
assert (c.naive(5) // c.this()).execute(2) == 2
assert (c.naive(5) % c.this()).execute(2) == 1
