def test_right_join():
join1 = (
c.join(
c.item(0),
c.item(1),
c.and_(
c.LEFT == c.RIGHT,
c.LEFT + c.RIGHT < 10,
c.LEFT > 0,
),
how="right",
)
.as_type(list)
.gen_converter(debug=False)
)
assert join1([(0, 1, 2, 3, 3), (3, 3, 4, 5)]) == [
(3, 3),
(3, 3),
(3, 3),
(3, 3),
(None, 4),
(None, 5),
]
conv = (
c.join(
c.item("first"),
c.item("second"),
(
c.LEFT.item("name").call_method("lower")
== c.RIGHT.item("full_name").call_method("lower")
),
how="right",
)
.as_type(list)
.gen_converter(debug=False)
)
# fmt: off
assert conv(
{
"first": [
{"name": "JOHN"},
{"name": "bob"},
{"name": "ron"},
],
"second": [
{"full_name": "BOB"},
{"full_name": "BOB"},
{"full_name": "John"},
{"full_name": "Nick"},
{"full_name": "Nick"},
],
}
) == [
({"name": "bob"}, {"full_name": "BOB"},),
({"name": "bob"}, {"full_name": "BOB"},),
({"name": "JOHN"}, {"full_name": "John"},),
(None, {"full_name": "Nick"}),
(None, {"full_name": "Nick"}),
]
def test_nested_loop_joins():
from convtools.conversion import join
join1 = (
join(
c.item(0),
c.item(1),
c.and_(
c.LEFT.item("id") - 1 == c.RIGHT.item("ID"),
c.LEFT.item("value") + 100 < c.RIGHT.item("value"),
c.LEFT.item("value") > 101,
c.RIGHT.item("value") < 209,
c.input_arg("flag"),
),
)
.as_type(list)
.gen_converter(debug=True)
)
assert join1(
[
[{"id": i, "value": i + 100} for i in range(10)],
[{"ID": i, "value": 210 - i} for i in range(10)],
],
flag=True,
) == [
({"id": 3, "value": 103}, {"ID": 2, "value": 208}),
({"id": 4, "value": 104}, {"ID": 3, "value": 207}),
({"id": 5, "value": 105}, {"ID": 4, "value": 206}),
]
join2 = (
join(
c.item(0),
c.item(1),
c.and_(c.LEFT * c.RIGHT < 30, c.LEFT + c.RIGHT > 8),
)
.as_type(list)
.gen_converter(debug=False)
)
assert join2([range(4, 10), range(4, 10)]) == [
(4, 5),
(4, 6),
(4, 7),
(5, 4),
(5, 5),
(6, 4),
(7, 4),
]
join3 = (
join(c.item(0), c.item(1), Eq(c.LEFT + c.RIGHT, 1))
.as_type(list)
.gen_converter(debug=True)
)
assert join3(([-1, 0, 1], [2, 1, 1])) == [(-1, 2), (0, 1), (0, 1)]
def test_outer_join():
join1 = (
c.join(
c.item(0),
c.item(1),
Eq(c.LEFT, c.RIGHT, 2),
how="full",
)
.as_type(list)
.gen_converter(debug=False)
)
assert join1(([0, 1, 2, 5], [2, 3, 4, 5])) == [
(0, None),
(1, None),
(2, 2),
(5, None),
(None, 3),
(None, 4),
(None, 5),
]
join2 = (
c.join(
c.item(0),
c.item(1),
c.and_(
c.LEFT == c.RIGHT,
c.LEFT + c.RIGHT < 10,
c.LEFT > 0,
),
how="outer",
)
.as_type(list)
.gen_converter(debug=False)
)
assert join2([(10, 7, 8, 0, 1, 2, 3, 3), (3, 3, 4, 5, 8)]) == [
(10, None),
(7, None),
(8, None),
(0, None),
(1, None),
(2, None),
(3, 3),
(3, 3),
(3, 3),
(3, 3),
(None, 4),
(None, 5),
(None, 8),
]
def test_join_with_input_args():
assert (c.join(
c.input_arg("custom_left"),
c.input_arg("custom_right"),
c.LEFT == c.RIGHT,
).as_type(list).execute(None,
custom_left=range(3),
custom_right=range(3),
debug=True)) == [(0, 0), (1, 1), (2, 2)]
def test_cross_join():
join1 = (c.join(c.item(0), c.item(1),
True).as_type(list).gen_converter(debug=False))
assert join1(([1, 2, 3], [5, 6])) == [
(1, 5),
(1, 6),
(2, 5),
(2, 6),
(3, 5),
(3, 6),
]
def test_right_join():
join1 = (c.join(
c.item(0),
c.item(1),
c.and_(
c.LEFT == c.RIGHT,
c.LEFT + c.RIGHT < 10,
c.LEFT > 0,
),
how="right",
).as_type(list).gen_converter(debug=False))
assert join1([(0, 1, 2, 3, 3), (3, 3, 4, 5)]) == [
(3, 3),
(3, 3),
(3, 3),
(3, 3),
(None, 4),
(None, 5),
]
def test_join_with_complex_pipe():
def f(l):
return l + [1, 3]
pipeline = (c.aggregate(c.ReduceFuncs.Array(c.item("a"))).pipe(
c.join(c.this(), c.call_func(f, c.this()),
c.LEFT == c.RIGHT)).iter(c.item(1)).as_type(list))
assert (pipeline.execute([
{
"a": 1
},
{
"a": 2
},
{
"a": 3
},
]) == [1, 1, 2, 3, 3])
def test_left_join():
join1 = (
join(
c.item(0),
c.item(1),
c.and_(c.LEFT == c.RIGHT, c.LEFT + c.RIGHT < 10, c.LEFT > 0,),
how="left",
)
.as_type(list)
.gen_converter(debug=True)
)
assert join1([(0, 1, 2, 3, 3), (3, 3, 4, 5)]) == [
(0, None),
(1, None),
(2, None),
(3, 3),
(3, 3),
(3, 3),
(3, 3),
]
def test_join_bad_inputs():
with pytest.raises(ValueError):
c.join(c.item(0), c.item(1), True, "hz")
def test_hash_joins():
join1 = (c.join(
c.item(0), c.item(1),
c.LEFT.item("id") == c.RIGHT.item("id")).as_type(list).gen_converter(
debug=False))
join1([
[{
"id": i,
"value": i + 100
} for i in range(3)],
[{
"id": i,
"value": i + 200
} for i in range(3)],
]) == [
({
"id": 0,
"value": 100
}, {
"id": 0,
"value": 200
}),
({
"id": 1,
"value": 101
}, {
"id": 1,
"value": 201
}),
({
"id": 2,
"value": 102
}, {
"id": 2,
"value": 202
}),
]
join2 = (c.join(
c.item(0),
c.item(1),
c.and_(
c.LEFT.item("id") == c.RIGHT.item("ID"),
c.LEFT.item("value") > 105,
c.RIGHT.item("value") < 209,
c.input_arg("flag"),
),
).as_type(list).gen_converter(debug=False))
assert join2(
[
[{
"id": i,
"value": i + 100
} for i in range(10)],
[{
"ID": i,
"value": i + 200
} for i in range(10)],
],
flag=True,
) == [
({
"id": 6,
"value": 106
}, {
"ID": 6,
"value": 206
}),
({
"id": 7,
"value": 107
}, {
"ID": 7,
"value": 207
}),
({
"id": 8,
"value": 108
}, {
"ID": 8,
"value": 208
}),
]
assert (join2(
[
[{
"id": i,
"value": i + 100
} for i in range(10)],
[{
"ID": i,
"value": i + 200
} for i in range(10)],
],
flag=False,
) == [])
join2 = (c.join(
c.item(0),
c.item(1),
c.and_(
c.LEFT.item("id") == c.RIGHT.item("ID"),
c.LEFT.item("value") > 105,
c.RIGHT.item("value") < 209,
c.input_arg("flag"),
),
).as_type(list).gen_converter(debug=False))
assert join2(
[
[{
"id": i,
"value": i + 100
} for i in range(10)],
[{
"ID": i,
"value": i + 200
} for i in range(10)],
],
flag=True,
) == [
({
"id": 6,
"value": 106
}, {
"ID": 6,
"value": 206
}),
({
"id": 7,
"value": 107
}, {
"ID": 7,
"value": 207
}),
({
"id": 8,
"value": 108
}, {
"ID": 8,
"value": 208
}),
]
def test_doc__index_intro():
# ======== #
# GROUP BY #
# ======== #
input_data = [
{
"a": 5,
"b": "foo"
},
{
"a": 10,
"b": "foo"
},
{
"a": 10,
"b": "bar"
},
{
"a": 10,
"b": "bar"
},
{
"a": 20,
"b": "bar"
},
]
conv = (c.group_by(c.item("b")).aggregate({
"b":
c.item("b"),
"a_first":
c.ReduceFuncs.First(c.item("a")),
"a_max":
c.ReduceFuncs.Max(c.item("a")),
}).gen_converter(debug=True))
assert conv(input_data) == [
{
"b": "foo",
"a_first": 5,
"a_max": 10
},
{
"b": "bar",
"a_first": 10,
"a_max": 20
},
]
# ========= #
# AGGREGATE #
# ========= #
conv = c.aggregate({
# list of "a" values where "b" equals to "bar"
"a":
c.ReduceFuncs.Array(c.item("a"), where=c.item("b") == "bar"),
# "b" value of a row where "a" has Max value
"b":
c.ReduceFuncs.MaxRow(c.item("a"), ).item("b", default=None),
}).gen_converter(debug=True)
assert conv(input_data) == {"a": [10, 10, 20], "b": "bar"}
# ==== #
# JOIN #
# ==== #
collection_1 = [
{
"id": 1,
"name": "Nick"
},
{
"id": 2,
"name": "Joash"
},
{
"id": 3,
"name": "Bob"
},
]
collection_2 = [
{
"ID": "3",
"age": 17,
"country": "GB"
},
{
"ID": "2",
"age": 21,
"country": "US"
},
{
"ID": "1",
"age": 18,
"country": "CA"
},
]
input_data = (collection_1, collection_2)
conv = (c.join(
c.item(0),
c.item(1),
c.and_(
c.LEFT.item("id") == c.RIGHT.item("ID").as_type(int),
c.RIGHT.item("age") >= 18,
),
how="left",
).pipe(
c.list_comp({
"id": c.item(0, "id"),
"name": c.item(0, "name"),
"age": c.item(1, "age", default=None),
"country": c.item(1, "country", default=None),
})).gen_converter(debug=True))
assert conv(input_data) == [
{
"id": 1,
"name": "Nick",
"age": 18,
"country": "CA"
},
{
"id": 2,
"name": "Joash",
"age": 21,
"country": "US"
},
{
"id": 3,
"name": "Bob",
"age": None,
"country": None
},
]
