def test_take_while(): result = c.take_while(c.this < 3).as_type(list).execute(range(5)) assert result == [0, 1, 2] result = ( c.call_func(range, c.this) .take_while(c.this < 3) .as_type(list) .execute(5) ) assert result == [0, 1, 2] def f(): yield from range(5) raise Exception result = ( c.take_while(c.this < c.input_arg("stop_before")) .filter(c.this >= c.input_arg("min_value")) .filter(c.this < 3, cast=list) .execute(f(), min_value=2, stop_before=4) ) assert result == [2] result = c.take_while(c.this < 0).as_type(list).execute(range(10)) assert result == []
def test_join_with_input_args(): assert (c.join( c.input_arg("custom_left"), c.input_arg("custom_right"), c.LEFT == c.RIGHT, ).as_type(list).execute(None, custom_left=range(3), custom_right=range(3), debug=True)) == [(0, 0), (1, 1), (2, 2)]
def test_pipe_label_args(): assert (c.this.pipe( c.this, label_input={ "label1": c.input_arg("abc") }, label_output={ "label2": c.input_arg("cde") }, ).execute(None, abc=1, cde=2) is None)
def test_optional_dict(): conv = c.list_comp({ "key1": c.item("key1"), "key2": c.optional(c.item("key2", default=None)), "key3": c.optional(c.item("key1") * 200, skip_value=2000), "key4": c.optional( c.item("key1") * c.input_arg("x") * 300, skip_if=c.item("key1") < 5, ), "key5": c.optional( c.item("key1") * c.input_arg("x") * 300, keep_if=c.item("key1") >= 5, ), c.optional(c.item("key2", default=-1), skip_value=-1): 0, c.optional(c.item("key1") * 400, skip_if=c.item("key1") < 5): c.optional(c.item("key22")), c.optional(c.item("key1") * 500, skip_if=c.item("key1") < 5): c.optional(c.item("key22"), skip_value=20), }).gen_converter(debug=False) assert conv([{ "key1": 1, "key2": 2 }, { "key1": 10, "key22": 20 }], x=1) == [ { "key1": 1, "key2": 2, "key3": 200, 2: 0 }, { "key1": 10, "key4": 3000, "key5": 3000, 4000: 20 }, ] with pytest.raises(Exception): c.list_comp(c.optional(c.item("key1"))).gen_converter() with pytest.raises(Exception): c.optional(c.item("key1"), skip_value=1, skip_if=c.this()) with pytest.raises(Exception): c.this().pipe(c.optional(c.this()))
class A: x = 10 def __init__(self): self.x = 20 conv1 = (c.this() + c.input_arg("self").attr("x")).gen_converter(method=True) conv2 = (c.this() + c.input_arg("cls").attr("x")).gen_converter(method=True) conv3 = classmethod( (c.this() + c.input_arg("cls").attr("x")).gen_converter(class_method=True)) conv4 = classmethod( (c.this() + c.input_arg("self").attr("x")).gen_converter(class_method=True)) conv5 = (c.this() + c.input_arg("self").attr("x") + c.input_arg("n")).gen_converter( signature="self, n=1000, data_=15") conv6 = staticmethod( ((c.this() + c.call_func(sum, c.input_arg("args"))) * c.input_arg("kwargs").call_method("get", "multiplicator", 1) ).gen_converter(signature="data_, *args, **kwargs"))
def test_labels(): conv1 = c.if_( 1, c.input_arg("y").item("abc").add_label("abc").pipe( c.input_arg("x").pipe( c.inline_expr("{cde} + 10").pass_args( cde=c.this().item("cde")))).pipe( c.inline_expr("{this} + {abc}").pass_args( this=c.this(), abc=c.label("abc"))), 2, ).gen_converter(debug=False) assert conv1(data_=1, x={"cde": 2}, y={"abc": 3}) == 15 list(c.generator_comp(c.this().add_label("a")).execute([1, 2])) c.list_comp(c.this().add_label("a")).execute([1, 2])
def test_pipes(): assert c.list_comp(c.inline_expr("{0} ** 2").pass_args(c.this())).pipe( c.call_func(sum, c.this())).pipe( c.call_func( lambda x, a: x + a, c.this(), c.naive({ "abc": 10 }).item(c.input_arg("key_name")), )).pipe([c.this(), c.this()]).execute([1, 2, 3], key_name="abc", debug=False) == [ 24, 24, ] assert c.item(0).pipe( datetime.strptime, "%Y-%m-%d", ).pipe(c.call_func(lambda dt: dt.date(), c.this())).execute([ "2019-01-01", ], debug=False) == date(2019, 1, 1) assert c.item(0).pipe( datetime.strptime, "%Y-%m-%d", ).pipe(c.this().call_method("date")).execute([ "2019-01-01", ], debug=False) == date(2019, 1, 1) with pytest.raises(c.ConversionException): c.naive(True).pipe(c.item("key1", _predefined_input={"key1": 777}))
def test_input_arg(): assert c.input_arg("x").as_type(int).execute(None, x="10") == 10 assert ( c.inline_expr(""""{{}}_{{}}".format(type({x}).__name__, {x})""") .pass_args(x=c.item("value")) .gen_converter() )({"value": 123}) == "int_123"
def test_simple_label(): conv1 = (c.tuple(c.item(2).add_label("a"), c.this()).pipe( c.item(1).pipe(c.list_comp( (c.this(), c.label("a"))))).gen_converter(debug=False)) assert conv1([1, 2, 3, 4]) == [(1, 3), (2, 3), (3, 3), (4, 3)] conv2 = (c.tuple(c.item(1).add_label("a"), c.this()).pipe( c.item(1), label_input={ "aa": c.item(0), "bb": c.item(0) }, label_output="collection1", ).pipe( c.label("collection1").pipe( c.aggregate( c.ReduceFuncs.Sum( c.this() + c.label("a") + c.label("aa") + c.input_arg("x") + c.label("collection1").item(0), ))), label_output="b", ).pipe(c.this() + c.label("b")).gen_converter(debug=False)) assert conv2([1, 2, 3, 4], x=10) == 140 conv3 = (c.tuple(c.item("default").add_label("default"), c.this()).pipe( c.item(1).pipe(c.item( "abc", default=c.label("default")))).gen_converter(debug=False)) assert conv3({"default": 1}) == 1 with pytest.raises(c.ConversionException): c.this().pipe(c.this(), label_input=1)
def test_slices(): assert c.this()[c.item(0):c.input_arg("slice_to"):c.item(1)].gen_converter( debug=False)([2, 2, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], slice_to=8) == [ 1, 3, 5, ]
def test_slices(): assert c.this[c.item(0):c.input_arg("slice_to"):c.item(1)].execute( [2, 2, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], slice_to=8) == [ 1, 3, 5, ]
def test_group_by_with_double_ended_pipes(): input_data = [ { "value": 1 }, { "value": 2 }, ] # fmt: off conv = c.aggregate( c.item("value").pipe(c.ReduceFuncs.Sum(c.this())).pipe( c.this() * 2)).gen_converter() # fmt: on result = conv(input_data) assert result == 6 input_data = [ { "k": "A", "v": 1 }, { "k": "A", "v": 2 }, ] reducer = c.ReduceFuncs.Sum(c.item("v")) conv = (c.group_by(c.item("k")).aggregate({ "v1": c.input_arg("test").pipe(reducer), "v2": reducer, }).gen_converter()) assert conv(input_data, test={"v": 7}) == [{"v1": 14, "v2": 3}]
def test_bad_namespace_usage(): with pytest.raises(Exception, match="rendering prevented by parent NamespaceCtx"): assert BadDropWhile(c.this < 100).gen_converter() with pytest.raises(Exception, match="rendering prevented by parent NamespaceCtx"): assert BadDropWhile(c.this < c.input_arg("abc")).gen_converter()
def test_nested_loop_joins(): from convtools.conversion import join join1 = ( join( c.item(0), c.item(1), c.and_( c.LEFT.item("id") - 1 == c.RIGHT.item("ID"), c.LEFT.item("value") + 100 < c.RIGHT.item("value"), c.LEFT.item("value") > 101, c.RIGHT.item("value") < 209, c.input_arg("flag"), ), ) .as_type(list) .gen_converter(debug=True) ) assert join1( [ [{"id": i, "value": i + 100} for i in range(10)], [{"ID": i, "value": 210 - i} for i in range(10)], ], flag=True, ) == [ ({"id": 3, "value": 103}, {"ID": 2, "value": 208}), ({"id": 4, "value": 104}, {"ID": 3, "value": 207}), ({"id": 5, "value": 105}, {"ID": 4, "value": 206}), ] join2 = ( join( c.item(0), c.item(1), c.and_(c.LEFT * c.RIGHT < 30, c.LEFT + c.RIGHT > 8), ) .as_type(list) .gen_converter(debug=False) ) assert join2([range(4, 10), range(4, 10)]) == [ (4, 5), (4, 6), (4, 7), (5, 4), (5, 5), (6, 4), (7, 4), ] join3 = ( join(c.item(0), c.item(1), Eq(c.LEFT + c.RIGHT, 1)) .as_type(list) .gen_converter(debug=True) ) assert join3(([-1, 0, 1], [2, 1, 1])) == [(-1, 2), (0, 1), (0, 1)]
def test_manually_defined_reducers(): data = [ { "name": "John", "category": "Games", "debit": 10, "balance": 90 }, { "name": "John", "category": "Games", "debit": 200, "balance": -110 }, { "name": "John", "category": "Food", "debit": 30, "balance": -140 }, { "name": "John", "category": "Games", "debit": 300, "balance": 0 }, { "name": "Nick", "category": "Food", "debit": 7, "balance": 50 }, { "name": "Nick", "category": "Games", "debit": 18, "balance": 32 }, { "name": "Bill", "category": "Games", "debit": 18, "balance": 120 }, ] grouper = (c.group_by(c.item("name")).aggregate( c.reduce(lambda a, b: a + b, c.item(c.input_arg("group_key")), initial=0)).filter(c.this() > 20).gen_converter( signature="data_, group_key='debit'")) assert grouper(data) == [540, 25] assert grouper(data, group_key="balance") == [82, 120]
def test_pipes(): assert c.list_comp(c.inline_expr("{0} ** 2").pass_args(c.this)).pipe( c.call_func(sum, c.this)).pipe( c.call_func( lambda x, a: x + a, c.this, c.naive({ "abc": 10 }).item(c.input_arg("key_name")), )).pipe([c.this, c.this]).execute([1, 2, 3], key_name="abc", debug=False) == [ 24, 24, ] assert c.item(0).pipe(datetime.strptime, "%Y-%m-%d").pipe( c.call_func(lambda dt: dt.date(), c.this)).execute(["2019-01-01"], debug=False) == date(2019, 1, 1) assert c.item(0).pipe(datetime.strptime, "%Y-%m-%d").pipe( c.this.call_method("date")).execute(["2019-01-01"], debug=False) == date(2019, 1, 1) conv = c.dict_comp( c.item("name"), c.item("transactions").pipe( c.list_comp({ "id": c.item(0).as_type(str), "amount": c.item(1).pipe(c.if_(c.this, c.this.as_type(Decimal), None)), })), ).gen_converter(debug=False) assert conv([{ "name": "test", "transactions": [(0, 0), (1, 10)] }]) == { "test": [ { "id": "0", "amount": None }, { "id": "1", "amount": Decimal("10") }, ] } assert c.this.pipe(lambda it: it).filter( c.this).sort().as_type(list).execute((2, 1, 0)) == [1, 2]
def test_cloning_functionality(): conv1 = c.item("a") conv2 = c.item("c").pipe(c.item("b").pipe(conv1)) conv3 = c.item("d").pipe(c.item(c.input_arg("test")).pipe(conv1)) assert conv3.execute({"d": {"X": {"a": 2}}}, test="X") == 2 assert conv2.execute({"c": {"b": {"a": 1}}}) == 1 conv1 = c.item("a") conv2 = conv1.item("b") conv3 = conv1.item("c") assert conv3.execute({"a": {"c": 3}}) == 3 assert conv2.execute({"a": {"b": 2}}) == 2
def test_drop_while(): result = c.drop_while(c.this < 3).as_type(list).execute(range(5)) assert result == [3, 4] result = ( c.call_func(range, c.this) .drop_while(c.this < c.input_arg("min_value")) .as_type(list) .execute(5, min_value=3) ) assert result == [3, 4] result = c.drop_while(c.this >= 0).as_type(list).execute(range(10)) assert result == []
def test_mutation_item(): now = datetime.now() assert c.list_comp( { "name": c.item("fullName"), "age": c.item("age").as_type(int), "to_del": 1, } ).pipe( c.list_comp( c.call_func(lambda d: d, c.this).tap( c.Mut.set_item( "name_before", c.label("_input").item(0, "name") ), c.Mut.set_item("name", c.item("name").call_method("lower")), c.Mut.set_item( "name_after", c.label("_input").item(0, "name") ), c.Mut.set_item("_updated", c.input_arg("now")), c.Mut.set_item(c.item("age"), c.item("age") >= 18), c.Mut.del_item("to_del"), c.Mut.custom(c.this.call_method("update", {"to_add": 2})), c.this.call_method("update", {"to_add2": 4}), ) ), label_input="_input", ).execute( [{"fullName": "John", "age": "28"}], debug=False, now=now ) == [ { "name": "john", "name_after": "john", "name_before": "John", "age": 28, "_updated": now, 28: True, "to_add": 2, "to_add2": 4, } ] with pytest.raises(Exception): c.item(c.Mut.set_item("abc", "cde")) with pytest.raises(Exception): conversion = c.item(1) conversion.ensure_conversion( c.Mut.set_item("abc", "cde"), explicitly_allowed_cls=GetItem )
def test_is_independent(): assert c(0).is_independent() assert c(int).is_independent() assert c(int).call().is_independent() assert c.label("a").is_independent() assert c.inline_expr("{}()").pass_args(int).is_independent() assert c.escaped_string("int()").is_independent() assert c({"a": c.input_arg("key")}).is_independent() assert not c.iter({"a": 1}).is_independent() assert not c.this.is_independent() assert not c({"a": 1}).item("a").is_independent() assert not c({"a": 1}).item(c.item("a")).is_independent() assert not c.inline_expr("{}()").pass_args(c.this).is_independent() assert not c.aggregate({"a": 1}).is_independent() assert not c.this.add_label("a").is_independent() assert not c(int).call(c.item(0)).is_independent()
def test_namespaces(): with pytest.raises(ValueError): LazyEscapedString("abc").execute([1]) with pytest.raises(ValueError): Namespace(LazyEscapedString("abc"), name_to_code={ "abc": None }).execute([1]) assert (Namespace(LazyEscapedString("abc"), name_to_code={ "abc": True }).execute(1) == 1) assert (Namespace( c.input_arg("abc") + LazyEscapedString("abc"), name_to_code={ "abc": "abc" }, ).execute(0.1, abc=2) == 4) assert Namespace(c.item(1), {}).execute([0, 10]) == 10 assert (Namespace( Namespace( Namespace(LazyEscapedString("abc"), name_to_code={"abc": True }) # 1 + LazyEscapedString("abc") # 10 + LazyEscapedString("foo") # 1000 + c.item() * 0.1, # 0.1, name_to_code={"foo": "arg_foo2"}, ), name_to_code={ "abc": "arg_abc", "foo": "arg_foo" }, )).gen_converter( debug=False, signature="data_, arg_abc=10, arg_foo=100, arg_foo2=1000")(1) == 1011.1 assert (Namespace( c.call_func(list, (1, )).pipe( c.if_( c.this, c.this * LazyEscapedString("number"), c.this, )), { "number": "3" }, ).execute(None) == [1, 1, 1])
def test_iter_mut_method(): assert c.iter(c.item(0)).as_type(list).execute([[1], [2]]) == [1, 2] assert c.iter_mut(c.Mut.custom(c.this.call_method("append", 7))).as_type( list ).execute([[1], [2]]) == [[1, 7], [2, 7]] result = ( c.this.iter({"a": c.this}) .iter_mut( c.Mut.set_item("b", c.item("a") + 1), c.Mut.set_item("c", c.item("a") + 2), ) .iter_mut( c.Mut.set_item("d", c.item("a") + 3), ) .as_type(list) .execute([1, 2, 3], debug=False) ) assert result == [ {"a": 1, "b": 2, "c": 3, "d": 4}, {"a": 2, "b": 3, "c": 4, "d": 5}, {"a": 3, "b": 4, "c": 5, "d": 6}, ] result = ( c.group_by(c.item(0)) .aggregate( c( [ {c.item(0): c.item(1).pipe(c.ReduceFuncs.Max(c.this))}, {c.item(1).pipe(c.ReduceFuncs.Max(c.this)): c.item(0)}, ] ) .iter_mut( c.Mut.set_item( "x", c.call_func(sum, c.this.call_method("values")) + c.input_arg("base"), ) ) .as_type(tuple) ) .execute([(0, 1), (0, 2), (1, 7)], base=100, debug=False) ) assert result == [ ({0: 2, "x": 102}, {2: 0, "x": 100}), ({1: 7, "x": 107}, {7: 1, "x": 101}), ]
def test_hashes(): assert hash(c.input_arg("abc")) == hash(c.input_arg("abc")) assert hash(c.input_arg("abd")) != hash(c.input_arg("abc")) assert hash(c.inline_expr("abc")) == hash(c.inline_expr("abc")) assert hash(c.inline_expr("abd")) != hash(c.inline_expr("abc"))
def test_join_conditions(): join_conditions = _JoinConditions.from_condition(c.LEFT == c.RIGHT) assert (True and join_conditions.inner_loop_conditions == [] and join_conditions.left_collection_filters == [] and join_conditions.left_row_filters == [] and join_conditions.left_row_hashers == [c.LEFT] and join_conditions.pre_filter == [] and join_conditions.right_collection_filters == [] and join_conditions.right_row_filters == [] and join_conditions.right_row_hashers == [c.RIGHT]) join_conditions = _JoinConditions.from_condition( c.or_(c.LEFT == c.RIGHT, c.LEFT == c.RIGHT)) c11 = c.LEFT.item(0) c12 = c.RIGHT.item(1) c21 = c.LEFT.item(1) c22 = c.RIGHT.item(0) c13 = c.LEFT.item(2) > 10 c23 = c.RIGHT.item(2) < 10 c01 = c.input_arg("x") > 100 join_conditions = _JoinConditions.from_condition( c.and_(c11 == c12, c22 == c21, c13).and_(c23, c01)) assert (True and join_conditions.inner_loop_conditions == [] and join_conditions.left_collection_filters == [c13] and join_conditions.left_row_filters == [] and join_conditions.left_row_hashers == [c11, c21] and join_conditions.pre_filter == [c01] and join_conditions.right_collection_filters == [c23] and join_conditions.right_row_filters == [] and join_conditions.right_row_hashers == [c12, c22]) join_conditions = _JoinConditions.from_condition(c.and_( c11 == c12, c22 == c21, c13).and_(c23, c01), how="left") assert (True and join_conditions.inner_loop_conditions == [] and join_conditions.left_collection_filters == [] and join_conditions.left_row_filters == [c13] and join_conditions.left_row_hashers == [c11, c21] and join_conditions.pre_filter == [c01] and join_conditions.right_collection_filters == [c23] and join_conditions.right_row_filters == [] and join_conditions.right_row_hashers == [c12, c22]) join_conditions = _JoinConditions.from_condition(c.and_( c11 == c12, c22 == c21, c13).and_(c23, c01), how="right") assert (True and join_conditions.inner_loop_conditions == [] and join_conditions.left_collection_filters == [c13] and join_conditions.left_row_filters == [] and join_conditions.left_row_hashers == [c11, c21] and join_conditions.pre_filter == [c01] and join_conditions.right_collection_filters == [] and join_conditions.right_row_filters == [c23] and join_conditions.right_row_hashers == [c12, c22]) join_conditions = _JoinConditions.from_condition(c.and_( c11 == c12, c22 == c21, c13).and_(c23, c01), how="outer") assert (True and join_conditions.inner_loop_conditions == [] and join_conditions.left_collection_filters == [] and join_conditions.left_row_filters == [c13] and join_conditions.left_row_hashers == [c11, c21] and join_conditions.pre_filter == [c01] and join_conditions.right_collection_filters == [] and join_conditions.right_row_filters == [c23] and join_conditions.right_row_hashers == [c12, c22]) with pytest.raises(AssertionError): _JoinConditions.from_condition(c.and_(True, False), how="abc") c1 = c.LEFT != c.RIGHT join_conditions = _JoinConditions.from_condition(c1) assert (True and join_conditions.inner_loop_conditions == [c1] and join_conditions.left_collection_filters == [] and join_conditions.left_row_filters == [] and join_conditions.left_row_hashers == [] and join_conditions.pre_filter == [] and join_conditions.right_collection_filters == [] and join_conditions.right_row_filters == [] and join_conditions.right_row_hashers == []) cond = c.LEFT > c.RIGHT join_conditions = _JoinConditions.from_condition(cond) assert (True and join_conditions.inner_loop_conditions == [cond] and join_conditions.left_collection_filters == [] and join_conditions.left_row_filters == [] and join_conditions.left_row_hashers == [] and join_conditions.pre_filter == [] and join_conditions.right_collection_filters == [] and join_conditions.right_row_filters == [] and join_conditions.right_row_hashers == []) c1 = c.LEFT == 1 c2 = c.RIGHT == 1 c3 = c.input_arg("x") == 1 join_conditions = _JoinConditions.from_condition(c1.and_(c2, c3), how="outer") assert (True and join_conditions.inner_loop_conditions == [] and join_conditions.left_collection_filters == [] and join_conditions.left_row_filters == [c1] and join_conditions.left_row_hashers == [] and join_conditions.pre_filter == [c3] and join_conditions.right_collection_filters == [] and join_conditions.right_row_filters == [c2] and join_conditions.right_row_hashers == []) c1 = c.LEFT + c.RIGHT + 10 c2 = c.LEFT + 1 c3 = c.RIGHT + 1 join_conditions = _JoinConditions.from_condition(c.and_(c1, c2, c3)) assert (True and join_conditions.inner_loop_conditions == [c1] and join_conditions.left_collection_filters == [c2] and join_conditions.left_row_filters == [] and join_conditions.left_row_hashers == [] and join_conditions.pre_filter == [] and join_conditions.right_collection_filters == [c3] and join_conditions.right_row_filters == [] and join_conditions.right_row_hashers == [])
def test_hash_joins(): join1 = (c.join( c.item(0), c.item(1), c.LEFT.item("id") == c.RIGHT.item("id")).as_type(list).gen_converter( debug=False)) join1([ [{ "id": i, "value": i + 100 } for i in range(3)], [{ "id": i, "value": i + 200 } for i in range(3)], ]) == [ ({ "id": 0, "value": 100 }, { "id": 0, "value": 200 }), ({ "id": 1, "value": 101 }, { "id": 1, "value": 201 }), ({ "id": 2, "value": 102 }, { "id": 2, "value": 202 }), ] join2 = (c.join( c.item(0), c.item(1), c.and_( c.LEFT.item("id") == c.RIGHT.item("ID"), c.LEFT.item("value") > 105, c.RIGHT.item("value") < 209, c.input_arg("flag"), ), ).as_type(list).gen_converter(debug=False)) assert join2( [ [{ "id": i, "value": i + 100 } for i in range(10)], [{ "ID": i, "value": i + 200 } for i in range(10)], ], flag=True, ) == [ ({ "id": 6, "value": 106 }, { "ID": 6, "value": 206 }), ({ "id": 7, "value": 107 }, { "ID": 7, "value": 207 }), ({ "id": 8, "value": 108 }, { "ID": 8, "value": 208 }), ] assert (join2( [ [{ "id": i, "value": i + 100 } for i in range(10)], [{ "ID": i, "value": i + 200 } for i in range(10)], ], flag=False, ) == []) join2 = (c.join( c.item(0), c.item(1), c.and_( c.LEFT.item("id") == c.RIGHT.item("ID"), c.LEFT.item("value") > 105, c.RIGHT.item("value") < 209, c.input_arg("flag"), ), ).as_type(list).gen_converter(debug=False)) assert join2( [ [{ "id": i, "value": i + 100 } for i in range(10)], [{ "ID": i, "value": i + 200 } for i in range(10)], ], flag=True, ) == [ ({ "id": 6, "value": 106 }, { "ID": 6, "value": 206 }), ({ "id": 7, "value": 107 }, { "ID": 7, "value": 207 }), ({ "id": 8, "value": 108 }, { "ID": 8, "value": 208 }), ]
def test_grouping(): data = [ { "name": "John", "category": "Games", "debit": 10, "balance": 90 }, { "name": "John", "category": "Games", "debit": 200, "balance": -110 }, { "name": "John", "category": "Food", "debit": 30, "balance": -140 }, { "name": "John", "category": "Games", "debit": 300, "balance": 0 }, { "name": "Nick", "category": "Food", "debit": 7, "balance": 50 }, { "name": "Nick", "category": "Games", "debit": 18, "balance": 32 }, { "name": "Bill", "category": "Games", "debit": 18, "balance": 120 }, ] result = (c.group_by(c.item("name")).aggregate(( c.item("name"), c.item("name").call_method("lower"), c.call_func(str.lower, c.item("name")), c.reduce( lambda a, b: a + b, c.item("debit"), initial=c.input_arg("arg1"), unconditional_init=True, ), c.reduce( c.inline_expr("{0} + {1}"), c.item("debit"), initial=lambda: 100, unconditional_init=True, ), c.reduce( max, c.item("debit"), prepare_first=lambda a: a, default=c.input_arg("arg1"), where=c.call_func(lambda x: x < 0, c.item("balance")), ), c.call_func( lambda max_debit, n: max_debit * n, c.reduce( max, c.item("debit"), prepare_first=lambda a: a, default=0, where=c.call_func(lambda x: x < 0, c.item("balance")), ), 1000, ), c.call_func( lambda max_debit, n: max_debit * n, c.reduce( c.ReduceFuncs.Max, c.item("debit"), default=1000, where=c.inline_expr("{0} > {1}").pass_args( c.item("balance"), c.input_arg("arg2"), ), ), -1, ), c.reduce(c.ReduceFuncs.MaxRow, c.item("debit")).item("balance"), c.reduce(c.ReduceFuncs.MinRow, c.item("debit")).item("balance"), )).sort(key=lambda t: t[0].lower(), reverse=True).execute(data, arg1=100, arg2=0, debug=False)) # fmt: off assert result == [ ('Nick', 'nick', 'nick', 125, 125, 100, 0, -18, 32, 50), ('John', 'john', 'john', 640, 640, 200, 200000, -10, 0, 90), ('Bill', 'bill', 'bill', 118, 118, 100, 0, -18, 120, 120), ] # fmt: on with pytest.raises(c.ConversionException): # there's a single group by field, while we use separate items # of this tuple in aggregate result = (c.group_by(c.item("name")).aggregate(( c.item("category"), c.reduce(c.ReduceFuncs.Sum, c.item("debit")), )).execute(data, debug=False)) aggregation = { c.call_func( tuple, c.ReduceFuncs.Array(c.item("name"), default=None), ): c.item("category").call_method("lower"), "count": c.ReduceFuncs.Count(), "max": c.ReduceFuncs.Max(c.item("debit")), "min": c.ReduceFuncs.Min(c.item("debit")), "count_distinct": c.ReduceFuncs.CountDistinct(c.item("name")), "array_agg_distinct": c.ReduceFuncs.ArrayDistinct(c.item("name")), "dict": c.ReduceFuncs.Dict(c.item("debit"), c.item("name")), } result = (c.group_by(c.item("category")).aggregate(aggregation).execute( data, debug=False)) result2 = (c.group_by(c.item("category")).aggregate( c.dict(*aggregation.items())).execute(data, debug=False)) # fmt: off assert result == result2 == [ { 'array_agg_distinct': ['John', 'Nick', 'Bill'], 'count': 5, 'count_distinct': 3, 'dict': { 10: 'John', 18: 'Bill', 200: 'John', 300: 'John' }, 'max': 300, 'min': 10, ('John', 'John', 'John', 'Nick', 'Bill'): 'games' }, { 'array_agg_distinct': ['John', 'Nick'], 'count': 2, 'count_distinct': 2, 'dict': { 7: 'Nick', 30: 'John' }, 'max': 30, 'min': 7, ('John', 'Nick'): 'food' } ] # fmt: on result3 = (c.aggregate(c.ReduceFuncs.Sum(c.item("debit"))).pipe( c.inline_expr("{0} + {1}").pass_args(c.this(), c.this())).execute(data, debug=False)) assert result3 == 583 * 2 by = c.item("name"), c.item("category") result4 = (c.group_by( *by).aggregate(by + (c.ReduceFuncs.Sum(c.item("debit")), )).execute( data, debug=False)) # fmt: off assert result4 == [('John', 'Games', 510), ('John', 'Food', 30), ('Nick', 'Food', 7), ('Nick', 'Games', 18), ('Bill', 'Games', 18)] # fmt: on result5 = (c.group_by().aggregate(c.ReduceFuncs.Sum( c.item("debit"))).execute(data, debug=False)) assert result5 == 583 with pytest.raises(c.ConversionException): # there's a single group by field, while we use separate items # of this tuple in aggregate (c.group_by(by).aggregate( by + (c.reduce(c.ReduceFuncs.Sum, c.item("debit")), )).execute( data, debug=False))
def test_gen_converter(): class A: x = 10 def __init__(self): self.x = 20 conv1 = (c.this() + c.input_arg("self").attr("x")).gen_converter(method=True) conv2 = (c.this() + c.input_arg("cls").attr("x")).gen_converter(method=True) conv3 = classmethod( (c.this() + c.input_arg("cls").attr("x")).gen_converter(class_method=True)) conv4 = classmethod( (c.this() + c.input_arg("self").attr("x")).gen_converter(class_method=True)) conv5 = (c.this() + c.input_arg("self").attr("x") + c.input_arg("n")).gen_converter( signature="self, n=1000, data_=15") conv6 = staticmethod( ((c.this() + c.call_func(sum, c.input_arg("args"))) * c.input_arg("kwargs").call_method("get", "multiplicator", 1) ).gen_converter(signature="data_, *args, **kwargs")) assert A().conv1(100) == 120 assert A.conv3(100) == 110 with pytest.raises(NameError): A().conv2(100) with pytest.raises(NameError): A.conv4(100) assert A().conv5() == 1035 assert A().conv5(data_=7) == 1027 assert A().conv5(n=100) == 135 assert A.conv6(20) == 20 assert A.conv6(20, 1, 2, 3) == 26 assert A.conv6(20, 1, 2, 3, multiplicator=10) == 260 assert (c.call_func(sum, c.this()).gen_converter(signature="*data_")(1, 2, 3) == 6) assert (c.call_func(lambda i: globals().__setitem__("A", 1) or sum(i), c.this()).gen_converter(signature="*data_")(1, 2, 3) == 6) assert c({ c.naive("-").call_method("join", c.this().call_method("keys")): c.call_func(sum, c.this().call_method("values")) }).gen_converter(signature="**data_")(a=1, b=2, c=3) == { "a-b-c": 6 } with pytest.raises(c.ConversionException): c.call_func(sum, c.input_arg("x")).gen_converter(signature="*data_")(1, 2, 3) with pytest.raises(c.ConversionException): c.this().gen_converter(method=True, class_method=True)
def test_conversions_dependencies(): input_arg = c.input_arg("abc") conv = c.item(input_arg) assert tuple(conv.get_dependencies()) == (input_arg, conv)
def test_doc__index_word_count(): # Let's say we need to count words across all files input_data = [ "war-and-peace-1.txt", "war-and-peace-2.txt", "war-and-peace-3.txt", "war-and-peace-4.txt", ] # # iterate an input and read file lines # # def read_file(filename): # with open(filename) as f: # for line in f: # yield line # extract_strings = c.generator_comp(c.call_func(read_file, c.this())) # to simplify testing extract_strings = c.generator_comp( c.call_func(lambda filename: [filename], c.this())) # 1. make ``re`` pattern available to the code to be generated # 2. call ``finditer`` method of the pattern and pass the string # as an argument # 3. pass the result to the next conversion # 4. iterate results, call ``.group()`` method of each re.Match # and call ``.lower()`` on each result split_words = (c.naive(re.compile(r"\w+")).call_method( "finditer", c.this()).pipe( c.generator_comp(c.this().call_method("group", 0).call_method("lower")))) # ``extract_strings`` is the generator of strings # so we iterate it and pass each item to ``split_words`` conversion vectorized_split_words = c.generator_comp(c.this().pipe(split_words)) # flattening the result of ``vectorized_split_words``, which is # a generator of generators of strings flatten = c.call_func( chain.from_iterable, c.this(), ) # aggregate the input, the result is a single dict # words are keys, values are count of words dict_word_to_count = c.aggregate( c.ReduceFuncs.DictCount(c.this(), c.this(), default=dict)) # take top N words by: # - call ``.items()`` method of the dict (the result of the aggregate) # - pass the result to ``sorted`` # - take the slice, using input argument named ``top_n`` # - cast to a dict take_top_n = (c.this().call_method("items").sort( key=lambda t: t[1], reverse=True).pipe(c.this()[:c.input_arg("top_n")]).as_type(dict)) # the resulting pipeline is pretty self-descriptive, except the ``c.if_`` # part, which checks the condition (first argument), # and returns the 2nd if True OR the 3rd (input data by default) otherwise pipeline = ( extract_strings.pipe(flatten).pipe(vectorized_split_words).pipe( flatten).pipe(dict_word_to_count).pipe( c.if_( c.input_arg("top_n").is_not(None), c.this().pipe(take_top_n), )) # Define the resulting converter function signature. In fact this # isn't necessary if you don't need to specify default values ).gen_converter(debug=True, signature="data_, top_n=None") assert pipeline(input_data, top_n=3) == {"war": 4, "and": 4, "peace": 4}
def test_naive_conversion_item(): d = {1: 2, 10: {"test": 15, 2: 777}, 100: {"test2": 200}} assert c.naive(d).item(1).execute(100) == 2 assert c.item(1).gen_converter()(d) == 2 assert c.item(10, "test").gen_converter()(d) == 15 assert c.item(11, "test", default=77).gen_converter()(d) == 77 assert (c.item(10, c.input_arg("arg1"), default=c.input_arg("arg2")).gen_converter()(d, arg1="test", arg2=77) == 15) assert (c.item(10, c.input_arg("arg1"), default=c.input_arg("arg2")).gen_converter()(d, arg1="tst", arg2=77) == 77) assert (c.item(11, "test", default=77).gen_converter(method=True)(None, d) == 77) assert c.item(10, "testt", default=77).gen_converter()(d) == 77 assert c.item(10, "testt", default=c.this()).gen_converter()(d) == d assert c.item(10, c.item(1)).gen_converter()(d) == 777 assert c.item(10).item(2).gen_converter()(d) == 777 with pytest.raises(KeyError): c.naive(d).item(11).gen_converter()(100) with pytest.raises(IndexError): c.naive([]).item(11).gen_converter()(100) with pytest.raises(TypeError): c.naive(None).item(11).gen_converter()(100) assert (c.naive(d).item(100).item("test2").gen_converter( debug=False)(100) == 200) assert (c.naive(d).item(c.this(), "test2").gen_converter(debug=False)(100) == 200) assert (c.naive(d).item(100, default=30).item( "test2", default=30).gen_converter(debug=False)(100) == 200) # testing defaults assert (c.naive(d).item(100, default=30).item( "test", default=30).gen_converter()(100) == 30) assert (c.naive(d).item(10).item("test2", default=30).gen_converter()(100) == 30) assert c.naive(True).is_(True).execute(100) is True assert c.naive(True).is_not(True).execute(100) is False assert c.naive(1).in_({1, 2}).execute(100) is True assert c.naive(1).in_({3, 2}).execute(100) is False assert c.naive(1).not_in({3, 2}).execute(100) is True assert c.naive(1).eq(1).execute(100) is True assert (c.naive(1) == 1).execute(100) is True assert c.naive(1).not_eq(1).execute(100) is False assert (c.naive(1) != 1).execute(100) is False assert c.naive(1).gte(1).execute(100) is True assert (c.naive(1) >= 1).execute(100) is True assert c.naive(2).gte(1).execute(100) is True assert (c.naive(2) >= 1).execute(100) is True assert c.naive(10).gt(1).execute(100) is True assert (c.naive(10) > 1).execute(100) is True assert c.naive(1).lte(1).execute(100) is True assert (c.naive(1) <= 1).execute(100) is True assert c.naive(0).lte(1).execute(100) is True assert (c.naive(0) <= 1).execute(100) is True assert c.naive(0).lt(1).execute(100) is True assert (c.naive(0) < 1).execute(100) is True assert c.this().neg().execute(2) == -2 assert (-c.this()).execute(2) == -2 assert (c.this() + c.this()).execute(2) == 4 assert (c.this() * c.this()).execute(3) == 9 assert (c.this() - c.this()).execute(2) == 0 assert (c.naive(5) / c.this()).execute(2) == 2.5 assert (c.naive(5) // c.this()).execute(2) == 2 assert (c.naive(5) % c.this()).execute(2) == 1