def test_item_attr_caching():
result = c({
"item": c.item(0).pipe(c.item(0, default=None)),
"item2": c.item(0).pipe(c.item(0, 1, default=None)),
"item3": c.item(0, default=-1).item(0, default=-2),
"item4": c.item(2, default=-1).item(0, default=-2),
"attr": c.item(1).pipe(c.attr("year", default=None)),
"attr2": c.item(1).pipe(c.attr("year", "month", default=None)),
"attr3": c.item(1, default=-1).attr("year", default=-2),
"attr4": c.item(2, default=-1).attr("year", default=-2),
}).execute([[1], date(1970, 1, 1)])
assert result == {
"item": 1,
"item2": None,
"item3": 1,
"item4": -2,
"attr": 1970,
"attr2": None,
"attr3": 1970,
"attr4": -2,
}
converter = (c.this.or_(None).item(
c.item("key"), default=c.item("default")).gen_converter())
assert converter({"key": "abc", "abc": 1, "default": -1}) == 1
assert converter({"key": "abc", "default": -1}) == -1
def test_list():
assert c.list(c.item(1), c.item(0), 3).gen_converter()([2, 1]) == [1, 2, 3]
assert c([[c.item(1), c.item(0), 3]]).gen_converter()([2, 1]) == [[
1,
2,
3,
]]
def test_dict():
assert c.dict((1, c.escaped_string("1+1")), (2, 3)).gen_converter()(
100
) == {1: 2, 2: 3}
assert c({1: c.escaped_string("1+1"), 2: 3}).gen_converter()(100) == {
1: 2,
2: 3,
}
def test_set():
assert c({c.item(1), c.item(0), 3}).gen_converter()([2, 1]) == {1, 2, 3}
assert c.set((c.item(1), c.item(0), 3)).gen_converter()([2, 1]) == {
(1, 2, 3)
}
assert c.set((c.item(1), c.item(0), 3)).gen_converter()([2, 1]) == {
(1, 2, 3)
}
def test_tuple():
assert c.tuple(c.item(1), c.item(0), 3).gen_converter()([2, 1]) == (
1,
2,
3,
)
assert c.tuple(
(c.item(1), c.item(0), 3)).gen_converter()([2, 1]) == ((1, 2, 3), )
assert c(()).execute(None) == ()
def test_base_reducer():
assert c.aggregate((
c.reduce(lambda a, b: a + b, c.this, initial=0),
c.reduce(c.naive(lambda a, b: a + b), c.this, initial=int),
c.reduce(
c.inline_expr("{0} + {1}"),
c.this,
initial=c.inline_expr("int()"),
default=0,
),
c.reduce(
c.inline_expr("{0} + {1}"),
c.this,
initial=c(int),
default=0,
),
c.reduce(
c.inline_expr("{0} + {1}"),
c.this,
initial=int,
default=0,
),
)).filter(c.this > 5).gen_converter(debug=False)([1, 2, 3]) == [
6,
6,
6,
6,
6,
]
with pytest.raises(ValueError):
c.aggregate(c.ReduceFuncs.Sum(c.reduce(
c.ReduceFuncs.Count))).gen_converter()
with pytest.raises(ValueError):
c.aggregate(c.ReduceFuncs.Sum(c.ReduceFuncs.Count() +
1)).gen_converter()
with pytest.raises(ValueError):
c.aggregate((c.ReduceFuncs.Count() +
2).pipe(c.ReduceFuncs.Sum(c.this) + 1)).gen_converter()
conv = c.aggregate(c.ReduceFuncs.DictArray(
c.item(0), c.item(1))).gen_converter(debug=False)
data = [
("a", 1),
("a", 2),
("b", 3),
]
result = {"a": [1, 2], "b": [3]}
assert conv(data) == result
assert conv([]) is None
conv2 = c.aggregate({
"key": c.ReduceFuncs.DictArray(c.item(0), c.item(1))
}).gen_converter(debug=False)
assert conv2([]) == {"key": None}
assert conv2(data) == {"key": result}
def test_min_max():
assert c.min(0, 1).execute(None) == 0
assert c.min(2, 1).execute(None) == 1
assert c.max(0, 1).execute(None) == 1
assert c.max(2, 1).execute(None) == 2
assert c.min(c.item(0), c.item(1)).execute((0, 1)) == 0
assert c((2, 1)).pipe(c.min(c.item(0), c.item(1))).execute(None) == 1
with pytest.raises(TypeError):
c.min(c.this).execute(-1)
with pytest.raises(TypeError):
c.max(c.this).execute(-1)
def test_iter_method():
assert (c.this.iter(c.this * 3).filter(c.this).as_type(list).execute(
[1, 2, 3, 0, 1],
debug=False,
) == [3, 6, 9, 3])
assert c.group_by(c.item(0)).aggregate(
c([
c.item(0),
c.item(1).pipe(c.ReduceFuncs.Max(c.this)),
]).iter(c.this * 100).as_type(tuple)).execute([(0, 1), (0, 2), (1, 7)],
debug=False) == [
(0, 200),
(100, 700),
]
def test_iter_mut_method():
assert c.iter(c.item(0)).as_type(list).execute([[1], [2]]) == [1, 2]
assert c.iter_mut(c.Mut.custom(c.this.call_method("append", 7))).as_type(
list
).execute([[1], [2]]) == [[1, 7], [2, 7]]
result = (
c.this.iter({"a": c.this})
.iter_mut(
c.Mut.set_item("b", c.item("a") + 1),
c.Mut.set_item("c", c.item("a") + 2),
)
.iter_mut(
c.Mut.set_item("d", c.item("a") + 3),
)
.as_type(list)
.execute([1, 2, 3], debug=False)
)
assert result == [
{"a": 1, "b": 2, "c": 3, "d": 4},
{"a": 2, "b": 3, "c": 4, "d": 5},
{"a": 3, "b": 4, "c": 5, "d": 6},
]
result = (
c.group_by(c.item(0))
.aggregate(
c(
[
{c.item(0): c.item(1).pipe(c.ReduceFuncs.Max(c.this))},
{c.item(1).pipe(c.ReduceFuncs.Max(c.this)): c.item(0)},
]
)
.iter_mut(
c.Mut.set_item(
"x",
c.call_func(sum, c.this.call_method("values"))
+ c.input_arg("base"),
)
)
.as_type(tuple)
)
.execute([(0, 1), (0, 2), (1, 7)], base=100, debug=False)
)
assert result == [
({0: 2, "x": 102}, {2: 0, "x": 100}),
({1: 7, "x": 107}, {7: 1, "x": 101}),
]
def test_group_by_with_pipes():
# fmt: off
input_data = [
{
"name": "John",
"started_at": date(2020, 1, 1),
"stopped_at": None,
"product": "A"
},
{
"name": "John",
"started_at": date(2020, 1, 1),
"stopped_at": date(2020, 1, 2),
"product": "B"
},
{
"name": "John",
"started_at": date(2020, 1, 1),
"stopped_at": None,
"product": "C"
},
{
"name": "Nick",
"started_at": date(2020, 1, 1),
"stopped_at": None,
"product": "D"
},
{
"name": "Nick",
"started_at": date(2020, 2, 1),
"stopped_at": None,
"product": "D"
},
{
"name": "Nick",
"started_at": date(2020, 2, 1),
"stopped_at": None,
"product": "E"
},
]
# fmt: on
output = (c.group_by(
c.item("name"),
c.item("started_at"),
).aggregate({
"name":
c.item("name"),
"started_at":
c.item("started_at"),
"products":
c.ReduceFuncs.ArrayDistinct(
c.if_(
c.item("stopped_at").is_(None),
c.item("product"),
None,
), ).pipe(c.filter(c.this())).pipe(
c.call_func(sorted, c.this()).pipe(
c(", ").call_method("join", c.this()))).pipe(c.this()),
}).execute(input_data))
# fmt: off
assert output == [{
'name': 'John',
'products': 'A, C',
'started_at': date(2020, 1, 1)
}, {
'name': 'Nick',
'products': 'D',
'started_at': date(2020, 1, 1)
}, {
'name': 'Nick',
'products': 'D, E',
'started_at': date(2020, 2, 1)
}]
# fmt: on
reducer = c.ReduceFuncs.Array(c.this(), default=list)
output = (c.group_by(
c.this()["name"],
c.this()["started_at"],
).aggregate({
"name": c.this()["name"],
"started_at": c.this()["started_at"],
"products": c.this()["product"].pipe(reducer)[:3],
}).execute(input_data))
assert output == [
{
"name": "John",
"products": ["A", "B", "C"],
"started_at": date(2020, 1, 1),
},
{
"name": "Nick",
"products": ["D"],
"started_at": date(2020, 1, 1),
},
{
"name": "Nick",
"products": ["D", "E"],
"started_at": date(2020, 2, 1),
},
]
def test_doc__index_deserialization():
class Employee:
def __init__(self, **kwargs):
self.kwargs = kwargs
input_data = {
"objects": [
{
"id": 1,
"first_name": "john",
"last_name": "black",
"dob": None,
"salary": "1,000.00",
"department": "D1 ",
"date": "2000-01-01",
},
{
"id": 2,
"first_name": "bob",
"last_name": "wick",
"dob": "1900-01-01",
"salary": "1,001.00",
"department": "D3 ",
"date": "2000-01-01",
},
]
}
# prepare a few conversions to reuse
c_strip = c.this.call_method("strip")
c_capitalize = c.this.call_method("capitalize")
c_decimal = c.this.call_method("replace", ",", "").as_type(Decimal)
c_date = c.call_func(datetime.strptime, c.this,
"%Y-%m-%d").call_method("date")
# reusing c_date
c_optional_date = c.if_(c.this, c_date, None)
first_name = c.item("first_name").pipe(c_capitalize)
last_name = c.item("last_name").pipe(c_capitalize)
# call "format" method of a string and pass first & last names as
# parameters
full_name = c("{} {}").call_method("format", first_name, last_name)
conv = (
c.item("objects").pipe(
c.generator_comp({
"id":
c.item("id"),
"first_name":
first_name,
"last_name":
last_name,
"full_name":
full_name,
"date_of_birth":
c.item("dob").pipe(c_optional_date),
"salary":
c.item("salary").pipe(c_decimal),
# pass a hardcoded dict and to get value by "department"
# key
"department_id":
c.naive({
"D1": 10,
"D2": 11,
"D3": 12,
}).item(c.item("department").pipe(c_strip)),
"date":
c.item("date").pipe(c_date),
})).pipe(
c.dict_comp(
c.item("id"), # key
c.apply_func( # value
Employee,
args=(),
kwargs=c.this,
),
)).gen_converter(debug=True) # to see print generated code
)
result = conv(input_data)
assert result[1].kwargs == {
"date": date(2000, 1, 1),
"date_of_birth": None,
"department_id": 10,
"first_name": "John",
"full_name": "John Black",
"id": 1,
"last_name": "Black",
"salary": Decimal("1000.00"),
}
assert result[2].kwargs == {
"date": date(2000, 1, 1),
"date_of_birth": date(1900, 1, 1),
"department_id": 12,
"first_name": "Bob",
"full_name": "Bob Wick",
"id": 2,
"last_name": "Wick",
"salary": Decimal("1001.00"),
}
def test_doc__index_deserialization():
class Employee:
def __init__(self, **kwargs):
self.kwargs = kwargs
input_data = {
"objects": [
{
"id": 1,
"first_name": "john",
"last_name": "black",
"dob": None,
"salary": "1,000.00",
"department": "D1 ",
"date": "2000-01-01",
},
{
"id": 2,
"first_name": "bob",
"last_name": "wick",
"dob": "1900-01-01",
"salary": "1,001.00",
"department": "D3 ",
"date": "2000-01-01",
},
]
}
# get by "department" key and then call method "strip"
department = c.item("department").call_method("strip")
first_name = c.item("first_name").call_method("capitalize")
last_name = c.item("last_name").call_method("capitalize")
# call "format" method of a string and pass first & last names as
# parameters
full_name = c("{} {}").call_method("format", first_name, last_name)
date_of_birth = c.item("dob")
# partially initialized "strptime"
parse_date = c.call_func(datetime.strptime, c.this(),
"%Y-%m-%d").call_method("date")
conv = (
c.item("objects").pipe(
c.generator_comp({
"id":
c.item("id"),
"first_name":
first_name,
"last_name":
last_name,
"full_name":
full_name,
"date_of_birth":
c.if_(
date_of_birth,
date_of_birth.pipe(parse_date),
None,
),
"salary":
c.call_func(
Decimal,
c.item("salary").call_method("replace", ",", ""),
),
# pass a hardcoded dict and to get value by "department"
# key
"department_id":
c.naive({
"D1": 10,
"D2": 11,
"D3": 12,
}).item(department),
"date":
c.item("date").pipe(parse_date),
})).
pipe(
c.dict_comp(
c.item(
"id"), # key
# write a python code expression, format with passed parameters
c.inline_expr("{employee_cls}(**{kwargs})").pass_args(
employee_cls=Employee,
kwargs=c.this(),
), # value
)).gen_converter(debug=True))
result = conv(input_data)
assert result[1].kwargs == {
"date": date(2000, 1, 1),
"date_of_birth": None,
"department_id": 10,
"first_name": "John",
"full_name": "John Black",
"id": 1,
"last_name": "Black",
"salary": Decimal("1000.00"),
}
assert result[2].kwargs == {
"date": date(2000, 1, 1),
"date_of_birth": date(1900, 1, 1),
"department_id": 12,
"first_name": "Bob",
"full_name": "Bob Wick",
"id": 2,
"last_name": "Wick",
"salary": Decimal("1001.00"),
}
def test_nested_group_by():
data = [
[0, [1, 2, 3]],
[0, [4, 5, 6]],
[1, [2, 3, 4]],
]
assert c.group_by(c.item(0)).aggregate((
c.item(0),
c.ReduceFuncs.Sum(
c.item(1).pipe(c.aggregate(c.ReduceFuncs.Sum(c.this())))),
)).execute(data, debug=False) == [
(0, 21),
(1, 9),
]
agg_conv = c.aggregate(c.ReduceFuncs.Sum(c.this()))
assert c.group_by(c.item(0)).aggregate((
c.item(0),
c.if_(
c.item(1),
c.item(1),
c.item(1),
).pipe(
c.if_(
c.this(),
c.this(),
c.this(),
).pipe(
c.ReduceFuncs.Sum(
c.if_(
c.this(),
c.this(),
c.this(),
).pipe((agg_conv, agg_conv)).pipe(c.item(1))).pipe(
c.if_(
c.this(),
c.this(),
c.this(),
)), )),
)).execute(data, debug=False) == [
(0, 21),
(1, 9),
]
summer = c.aggregate(c.ReduceFuncs.Sum(c.this()))
merger = c.aggregate({
"value1":
c.ReduceFuncs.First(c.item("value1"), where=c("value1").in_(c.this())),
"value2":
c.ReduceFuncs.First(c.item("value2"),
where=c("value2").in_(c.this())).pipe(
c.if_(c.this(),
c.this().pipe(summer))),
})
converter = (c.group_by(c.item("id_")).aggregate({
"id_":
c.item("id_"),
"data":
c.ReduceFuncs.Array(c.this()).pipe(merger),
}).gen_converter(debug=False))
assert converter([
{
"id_": 1,
"value1": 2
},
{
"id_": 2,
"value1": 3
},
{
"id_": 2,
"value2": [1, 2, 3]
},
]) == [
{
"id_": 1,
"data": {
"value1": 2,
"value2": None
}
},
{
"id_": 2,
"data": {
"value1": 3,
"value2": 6
}
},
]
def g():
yield 1
raise Exception
assert (c.aggregate(c.ReduceFuncs.First(c.this())).execute(
g(), debug=False)) == 1
class SumReducer4(MultiStatementReducer):
prepare_first = ("%(result)s = {0}", )
reduce = ("%(result)s = {prev_result} + ({0} or 4)", )
default = c(0)
unconditional_init = True
