def test_item_attr_caching(): result = c({ "item": c.item(0).pipe(c.item(0, default=None)), "item2": c.item(0).pipe(c.item(0, 1, default=None)), "item3": c.item(0, default=-1).item(0, default=-2), "item4": c.item(2, default=-1).item(0, default=-2), "attr": c.item(1).pipe(c.attr("year", default=None)), "attr2": c.item(1).pipe(c.attr("year", "month", default=None)), "attr3": c.item(1, default=-1).attr("year", default=-2), "attr4": c.item(2, default=-1).attr("year", default=-2), }).execute([[1], date(1970, 1, 1)]) assert result == { "item": 1, "item2": None, "item3": 1, "item4": -2, "attr": 1970, "attr2": None, "attr3": 1970, "attr4": -2, } converter = (c.this.or_(None).item( c.item("key"), default=c.item("default")).gen_converter()) assert converter({"key": "abc", "abc": 1, "default": -1}) == 1 assert converter({"key": "abc", "default": -1}) == -1
def test_list(): assert c.list(c.item(1), c.item(0), 3).gen_converter()([2, 1]) == [1, 2, 3] assert c([[c.item(1), c.item(0), 3]]).gen_converter()([2, 1]) == [[ 1, 2, 3, ]]
def test_dict(): assert c.dict((1, c.escaped_string("1+1")), (2, 3)).gen_converter()( 100 ) == {1: 2, 2: 3} assert c({1: c.escaped_string("1+1"), 2: 3}).gen_converter()(100) == { 1: 2, 2: 3, }
def test_set(): assert c({c.item(1), c.item(0), 3}).gen_converter()([2, 1]) == {1, 2, 3} assert c.set((c.item(1), c.item(0), 3)).gen_converter()([2, 1]) == { (1, 2, 3) } assert c.set((c.item(1), c.item(0), 3)).gen_converter()([2, 1]) == { (1, 2, 3) }
def test_tuple(): assert c.tuple(c.item(1), c.item(0), 3).gen_converter()([2, 1]) == ( 1, 2, 3, ) assert c.tuple( (c.item(1), c.item(0), 3)).gen_converter()([2, 1]) == ((1, 2, 3), ) assert c(()).execute(None) == ()
def test_base_reducer(): assert c.aggregate(( c.reduce(lambda a, b: a + b, c.this, initial=0), c.reduce(c.naive(lambda a, b: a + b), c.this, initial=int), c.reduce( c.inline_expr("{0} + {1}"), c.this, initial=c.inline_expr("int()"), default=0, ), c.reduce( c.inline_expr("{0} + {1}"), c.this, initial=c(int), default=0, ), c.reduce( c.inline_expr("{0} + {1}"), c.this, initial=int, default=0, ), )).filter(c.this > 5).gen_converter(debug=False)([1, 2, 3]) == [ 6, 6, 6, 6, 6, ] with pytest.raises(ValueError): c.aggregate(c.ReduceFuncs.Sum(c.reduce( c.ReduceFuncs.Count))).gen_converter() with pytest.raises(ValueError): c.aggregate(c.ReduceFuncs.Sum(c.ReduceFuncs.Count() + 1)).gen_converter() with pytest.raises(ValueError): c.aggregate((c.ReduceFuncs.Count() + 2).pipe(c.ReduceFuncs.Sum(c.this) + 1)).gen_converter() conv = c.aggregate(c.ReduceFuncs.DictArray( c.item(0), c.item(1))).gen_converter(debug=False) data = [ ("a", 1), ("a", 2), ("b", 3), ] result = {"a": [1, 2], "b": [3]} assert conv(data) == result assert conv([]) is None conv2 = c.aggregate({ "key": c.ReduceFuncs.DictArray(c.item(0), c.item(1)) }).gen_converter(debug=False) assert conv2([]) == {"key": None} assert conv2(data) == {"key": result}
def test_min_max(): assert c.min(0, 1).execute(None) == 0 assert c.min(2, 1).execute(None) == 1 assert c.max(0, 1).execute(None) == 1 assert c.max(2, 1).execute(None) == 2 assert c.min(c.item(0), c.item(1)).execute((0, 1)) == 0 assert c((2, 1)).pipe(c.min(c.item(0), c.item(1))).execute(None) == 1 with pytest.raises(TypeError): c.min(c.this).execute(-1) with pytest.raises(TypeError): c.max(c.this).execute(-1)
def test_iter_method(): assert (c.this.iter(c.this * 3).filter(c.this).as_type(list).execute( [1, 2, 3, 0, 1], debug=False, ) == [3, 6, 9, 3]) assert c.group_by(c.item(0)).aggregate( c([ c.item(0), c.item(1).pipe(c.ReduceFuncs.Max(c.this)), ]).iter(c.this * 100).as_type(tuple)).execute([(0, 1), (0, 2), (1, 7)], debug=False) == [ (0, 200), (100, 700), ]
def test_iter_mut_method(): assert c.iter(c.item(0)).as_type(list).execute([[1], [2]]) == [1, 2] assert c.iter_mut(c.Mut.custom(c.this.call_method("append", 7))).as_type( list ).execute([[1], [2]]) == [[1, 7], [2, 7]] result = ( c.this.iter({"a": c.this}) .iter_mut( c.Mut.set_item("b", c.item("a") + 1), c.Mut.set_item("c", c.item("a") + 2), ) .iter_mut( c.Mut.set_item("d", c.item("a") + 3), ) .as_type(list) .execute([1, 2, 3], debug=False) ) assert result == [ {"a": 1, "b": 2, "c": 3, "d": 4}, {"a": 2, "b": 3, "c": 4, "d": 5}, {"a": 3, "b": 4, "c": 5, "d": 6}, ] result = ( c.group_by(c.item(0)) .aggregate( c( [ {c.item(0): c.item(1).pipe(c.ReduceFuncs.Max(c.this))}, {c.item(1).pipe(c.ReduceFuncs.Max(c.this)): c.item(0)}, ] ) .iter_mut( c.Mut.set_item( "x", c.call_func(sum, c.this.call_method("values")) + c.input_arg("base"), ) ) .as_type(tuple) ) .execute([(0, 1), (0, 2), (1, 7)], base=100, debug=False) ) assert result == [ ({0: 2, "x": 102}, {2: 0, "x": 100}), ({1: 7, "x": 107}, {7: 1, "x": 101}), ]
def test_group_by_with_pipes(): # fmt: off input_data = [ { "name": "John", "started_at": date(2020, 1, 1), "stopped_at": None, "product": "A" }, { "name": "John", "started_at": date(2020, 1, 1), "stopped_at": date(2020, 1, 2), "product": "B" }, { "name": "John", "started_at": date(2020, 1, 1), "stopped_at": None, "product": "C" }, { "name": "Nick", "started_at": date(2020, 1, 1), "stopped_at": None, "product": "D" }, { "name": "Nick", "started_at": date(2020, 2, 1), "stopped_at": None, "product": "D" }, { "name": "Nick", "started_at": date(2020, 2, 1), "stopped_at": None, "product": "E" }, ] # fmt: on output = (c.group_by( c.item("name"), c.item("started_at"), ).aggregate({ "name": c.item("name"), "started_at": c.item("started_at"), "products": c.ReduceFuncs.ArrayDistinct( c.if_( c.item("stopped_at").is_(None), c.item("product"), None, ), ).pipe(c.filter(c.this())).pipe( c.call_func(sorted, c.this()).pipe( c(", ").call_method("join", c.this()))).pipe(c.this()), }).execute(input_data)) # fmt: off assert output == [{ 'name': 'John', 'products': 'A, C', 'started_at': date(2020, 1, 1) }, { 'name': 'Nick', 'products': 'D', 'started_at': date(2020, 1, 1) }, { 'name': 'Nick', 'products': 'D, E', 'started_at': date(2020, 2, 1) }] # fmt: on reducer = c.ReduceFuncs.Array(c.this(), default=list) output = (c.group_by( c.this()["name"], c.this()["started_at"], ).aggregate({ "name": c.this()["name"], "started_at": c.this()["started_at"], "products": c.this()["product"].pipe(reducer)[:3], }).execute(input_data)) assert output == [ { "name": "John", "products": ["A", "B", "C"], "started_at": date(2020, 1, 1), }, { "name": "Nick", "products": ["D"], "started_at": date(2020, 1, 1), }, { "name": "Nick", "products": ["D", "E"], "started_at": date(2020, 2, 1), }, ]
def test_doc__index_deserialization(): class Employee: def __init__(self, **kwargs): self.kwargs = kwargs input_data = { "objects": [ { "id": 1, "first_name": "john", "last_name": "black", "dob": None, "salary": "1,000.00", "department": "D1 ", "date": "2000-01-01", }, { "id": 2, "first_name": "bob", "last_name": "wick", "dob": "1900-01-01", "salary": "1,001.00", "department": "D3 ", "date": "2000-01-01", }, ] } # prepare a few conversions to reuse c_strip = c.this.call_method("strip") c_capitalize = c.this.call_method("capitalize") c_decimal = c.this.call_method("replace", ",", "").as_type(Decimal) c_date = c.call_func(datetime.strptime, c.this, "%Y-%m-%d").call_method("date") # reusing c_date c_optional_date = c.if_(c.this, c_date, None) first_name = c.item("first_name").pipe(c_capitalize) last_name = c.item("last_name").pipe(c_capitalize) # call "format" method of a string and pass first & last names as # parameters full_name = c("{} {}").call_method("format", first_name, last_name) conv = ( c.item("objects").pipe( c.generator_comp({ "id": c.item("id"), "first_name": first_name, "last_name": last_name, "full_name": full_name, "date_of_birth": c.item("dob").pipe(c_optional_date), "salary": c.item("salary").pipe(c_decimal), # pass a hardcoded dict and to get value by "department" # key "department_id": c.naive({ "D1": 10, "D2": 11, "D3": 12, }).item(c.item("department").pipe(c_strip)), "date": c.item("date").pipe(c_date), })).pipe( c.dict_comp( c.item("id"), # key c.apply_func( # value Employee, args=(), kwargs=c.this, ), )).gen_converter(debug=True) # to see print generated code ) result = conv(input_data) assert result[1].kwargs == { "date": date(2000, 1, 1), "date_of_birth": None, "department_id": 10, "first_name": "John", "full_name": "John Black", "id": 1, "last_name": "Black", "salary": Decimal("1000.00"), } assert result[2].kwargs == { "date": date(2000, 1, 1), "date_of_birth": date(1900, 1, 1), "department_id": 12, "first_name": "Bob", "full_name": "Bob Wick", "id": 2, "last_name": "Wick", "salary": Decimal("1001.00"), }
def test_doc__index_deserialization(): class Employee: def __init__(self, **kwargs): self.kwargs = kwargs input_data = { "objects": [ { "id": 1, "first_name": "john", "last_name": "black", "dob": None, "salary": "1,000.00", "department": "D1 ", "date": "2000-01-01", }, { "id": 2, "first_name": "bob", "last_name": "wick", "dob": "1900-01-01", "salary": "1,001.00", "department": "D3 ", "date": "2000-01-01", }, ] } # get by "department" key and then call method "strip" department = c.item("department").call_method("strip") first_name = c.item("first_name").call_method("capitalize") last_name = c.item("last_name").call_method("capitalize") # call "format" method of a string and pass first & last names as # parameters full_name = c("{} {}").call_method("format", first_name, last_name) date_of_birth = c.item("dob") # partially initialized "strptime" parse_date = c.call_func(datetime.strptime, c.this(), "%Y-%m-%d").call_method("date") conv = ( c.item("objects").pipe( c.generator_comp({ "id": c.item("id"), "first_name": first_name, "last_name": last_name, "full_name": full_name, "date_of_birth": c.if_( date_of_birth, date_of_birth.pipe(parse_date), None, ), "salary": c.call_func( Decimal, c.item("salary").call_method("replace", ",", ""), ), # pass a hardcoded dict and to get value by "department" # key "department_id": c.naive({ "D1": 10, "D2": 11, "D3": 12, }).item(department), "date": c.item("date").pipe(parse_date), })). pipe( c.dict_comp( c.item( "id"), # key # write a python code expression, format with passed parameters c.inline_expr("{employee_cls}(**{kwargs})").pass_args( employee_cls=Employee, kwargs=c.this(), ), # value )).gen_converter(debug=True)) result = conv(input_data) assert result[1].kwargs == { "date": date(2000, 1, 1), "date_of_birth": None, "department_id": 10, "first_name": "John", "full_name": "John Black", "id": 1, "last_name": "Black", "salary": Decimal("1000.00"), } assert result[2].kwargs == { "date": date(2000, 1, 1), "date_of_birth": date(1900, 1, 1), "department_id": 12, "first_name": "Bob", "full_name": "Bob Wick", "id": 2, "last_name": "Wick", "salary": Decimal("1001.00"), }
def test_nested_group_by(): data = [ [0, [1, 2, 3]], [0, [4, 5, 6]], [1, [2, 3, 4]], ] assert c.group_by(c.item(0)).aggregate(( c.item(0), c.ReduceFuncs.Sum( c.item(1).pipe(c.aggregate(c.ReduceFuncs.Sum(c.this())))), )).execute(data, debug=False) == [ (0, 21), (1, 9), ] agg_conv = c.aggregate(c.ReduceFuncs.Sum(c.this())) assert c.group_by(c.item(0)).aggregate(( c.item(0), c.if_( c.item(1), c.item(1), c.item(1), ).pipe( c.if_( c.this(), c.this(), c.this(), ).pipe( c.ReduceFuncs.Sum( c.if_( c.this(), c.this(), c.this(), ).pipe((agg_conv, agg_conv)).pipe(c.item(1))).pipe( c.if_( c.this(), c.this(), c.this(), )), )), )).execute(data, debug=False) == [ (0, 21), (1, 9), ] summer = c.aggregate(c.ReduceFuncs.Sum(c.this())) merger = c.aggregate({ "value1": c.ReduceFuncs.First(c.item("value1"), where=c("value1").in_(c.this())), "value2": c.ReduceFuncs.First(c.item("value2"), where=c("value2").in_(c.this())).pipe( c.if_(c.this(), c.this().pipe(summer))), }) converter = (c.group_by(c.item("id_")).aggregate({ "id_": c.item("id_"), "data": c.ReduceFuncs.Array(c.this()).pipe(merger), }).gen_converter(debug=False)) assert converter([ { "id_": 1, "value1": 2 }, { "id_": 2, "value1": 3 }, { "id_": 2, "value2": [1, 2, 3] }, ]) == [ { "id_": 1, "data": { "value1": 2, "value2": None } }, { "id_": 2, "data": { "value1": 3, "value2": 6 } }, ] def g(): yield 1 raise Exception assert (c.aggregate(c.ReduceFuncs.First(c.this())).execute( g(), debug=False)) == 1
class SumReducer4(MultiStatementReducer): prepare_first = ("%(result)s = {0}", ) reduce = ("%(result)s = {prev_result} + ({0} or 4)", ) default = c(0) unconditional_init = True