def test_pipe_filter_sort(): assert (c.this().as_type(list).pipe(c.iter(c.this() + 1)).filter( c.this() > 3).sort(key=lambda x: x, reverse=True).execute(range(7), debug=False)) == [7, 6, 5, 4] assert c.this().sort().execute([3, 1, 2]) == [1, 2, 3]
def test_linecache_cleaning(): _ConverterCallable.linecache_keys.max_size = 100 length_before = len(linecache.cache) for i in range(100): c.this().gen_converter(debug=False) length_after_100 = len(linecache.cache) for i in range(10): c.this().gen_converter(debug=False) length_after_110 = len(linecache.cache) assert (length_after_110 == length_after_100 and length_before + 100 >= length_after_100) for key in list(linecache.cache.keys()): del linecache.cache[key] converter_callable = c.this().gen_converter(debug=False) for ( fake_filename, code_str, ) in converter_callable._fake_filename_to_code_str.items(): converter_callable.add_sources(fake_filename, code_str) with pytest.raises(Exception): converter_callable.add_sources(fake_filename, code_str + " ")
def test_memory_freeing(): converter = ( c.this() .pipe( c.list_comp(c.this() + c.label("input_data").item(0)), label_input=dict(input_data=c.this()), ) .gen_converter(debug=True) ) sizes = [] sizes.append(total_size(converter.__dict__)) for i in range(100): l_input = [i + j for j in range(3)] l_out = [j + l_input[0] for j in l_input] assert converter(l_input) == l_out sizes.append(total_size(converter.__dict__)) assert all(sizes[0] == size for size in sizes[1:]), sizes conv2 = ( c.inline_expr("globals().__setitem__('a', {}) or 1") .pass_args(c.this()) .gen_converter() ) with pytest.raises(AssertionError): # should raise because of a memory leak conv2(123)
def test_group_by_reducer_clones(): data = [ { "value": 2 }, { "value": 3 }, ] conv = c.aggregate( c.item("value").pipe(c.ReduceFuncs.Sum(c.this()).pipe(c.this() + 1))) assert conv.execute(data) == 6 reducer = c.ReduceFuncs.DictSum(c.item("k"), c.item("v")) reducer1 = c.item("item1").pipe(reducer) reducer2 = c.item("item2").pipe(reducer) assert c.aggregate(reducer1).execute([{ "item1": { "k": 1, "v": 2 } }]) == { 1: 2 } assert c.aggregate(reducer2).execute([{ "item2": { "k": 2, "v": 3 } }]) == { 2: 3 }
def test_group_by_with_double_ended_pipes(): input_data = [ { "value": 1 }, { "value": 2 }, ] # fmt: off conv = c.aggregate( c.item("value").pipe(c.ReduceFuncs.Sum(c.this())).pipe( c.this() * 2)).gen_converter() # fmt: on result = conv(input_data) assert result == 6 input_data = [ { "k": "A", "v": 1 }, { "k": "A", "v": 2 }, ] reducer = c.ReduceFuncs.Sum(c.item("v")) conv = (c.group_by(c.item("k")).aggregate({ "v1": c.input_arg("test").pipe(reducer), "v2": reducer, }).gen_converter()) assert conv(input_data, test={"v": 7}) == [{"v1": 14, "v2": 3}]
def test_sort(): assert c.sort().execute([2, 3, 1]) == [1, 2, 3] assert c.sort(key=lambda x: x, reverse=True).execute([2, 3, 1]) == [ 3, 2, 1, ] assert c.this().sort().execute([2, 3, 1]) == [1, 2, 3] assert c.this().sort(key=lambda x: x, reverse=False).execute([2, 3, 1]) == [1, 2, 3]
def test_simple_label(): conv1 = (c.tuple(c.item(2).add_label("a"), c.this()).pipe( c.item(1).pipe(c.list_comp( (c.this(), c.label("a"))))).gen_converter(debug=False)) assert conv1([1, 2, 3, 4]) == [(1, 3), (2, 3), (3, 3), (4, 3)] conv2 = (c.tuple(c.item(1).add_label("a"), c.this()).pipe( c.item(1), label_input={ "aa": c.item(0), "bb": c.item(0) }, label_output="collection1", ).pipe( c.label("collection1").pipe( c.aggregate( c.ReduceFuncs.Sum( c.this() + c.label("a") + c.label("aa") + c.input_arg("x") + c.label("collection1").item(0), ))), label_output="b", ).pipe(c.this() + c.label("b")).gen_converter(debug=False)) assert conv2([1, 2, 3, 4], x=10) == 140 conv3 = (c.tuple(c.item("default").add_label("default"), c.this()).pipe( c.item(1).pipe(c.item( "abc", default=c.label("default")))).gen_converter(debug=False)) assert conv3({"default": 1}) == 1 with pytest.raises(c.ConversionException): c.this().pipe(c.this(), label_input=1)
def test_multiple_aggregations(dict_series): assert ( c.aggregate(c.ReduceFuncs.Array(c.item("name"))) .pipe( c.aggregate(c.ReduceFuncs.ArrayDistinct(c.this())).pipe( c.aggregate(c.ReduceFuncs.Max(c.this())) ) ) .execute(dict_series, debug=False) == "Nick" )
def test_optional_dict(): conv = c.list_comp({ "key1": c.item("key1"), "key2": c.optional(c.item("key2", default=None)), "key3": c.optional(c.item("key1") * 200, skip_value=2000), "key4": c.optional( c.item("key1") * c.input_arg("x") * 300, skip_if=c.item("key1") < 5, ), "key5": c.optional( c.item("key1") * c.input_arg("x") * 300, keep_if=c.item("key1") >= 5, ), c.optional(c.item("key2", default=-1), skip_value=-1): 0, c.optional(c.item("key1") * 400, skip_if=c.item("key1") < 5): c.optional(c.item("key22")), c.optional(c.item("key1") * 500, skip_if=c.item("key1") < 5): c.optional(c.item("key22"), skip_value=20), }).gen_converter(debug=False) assert conv([{ "key1": 1, "key2": 2 }, { "key1": 10, "key22": 20 }], x=1) == [ { "key1": 1, "key2": 2, "key3": 200, 2: 0 }, { "key1": 10, "key4": 3000, "key5": 3000, 4000: 20 }, ] with pytest.raises(Exception): c.list_comp(c.optional(c.item("key1"))).gen_converter() with pytest.raises(Exception): c.optional(c.item("key1"), skip_value=1, skip_if=c.this()) with pytest.raises(Exception): c.this().pipe(c.optional(c.this()))
def test_min_max(): assert c.min(0, 1).execute(None) == 0 assert c.min(2, 1).execute(None) == 1 assert c.max(0, 1).execute(None) == 1 assert c.max(2, 1).execute(None) == 2 assert c.min(c.item(0), c.item(1)).execute((0, 1)) == 0 assert c((2, 1)).pipe(c.min(c.item(0), c.item(1))).execute(None) == 1 with pytest.raises(TypeError): c.min(c.this()).execute(-1) with pytest.raises(TypeError): c.max(c.this()).execute(-1)
def test_legacy_dict_reduce_approach(dict_series): output = c.aggregate( c.reduce( c.ReduceFuncs.DictSum, (c.item("name"), c.item("value")), )).execute(dict_series) assert output == { "Nick": 3, "John": 63, } with pytest.raises(ValueError): c.ReduceFuncs.DictSum(c.this(), c.this(), c.this()) with pytest.raises(ValueError): c.ReduceFuncs.DictSum({c.this(), c.this()})
def test_caching_conversion(): class CustomException(Exception): pass def f(number): if not f.first_time: raise CustomException f.first_time = False return number f.first_time = True conv = (c.call_func(f, c.this()).pipe( c.if_(c.this(), c.this() + 1, c.this() + 2)).gen_converter()) assert conv(0) == 2 with pytest.raises(CustomException): assert conv(0) == 2 f.first_time = True assert conv(1) == 2 with pytest.raises(CustomException): c.call_func(f, c.this()).pipe( c.if_(c.this(), c.this() + 1, c.this() + 2, no_input_caching=True)).execute(0)
def test_labels(): conv1 = c.if_( 1, c.input_arg("y").item("abc").add_label("abc").pipe( c.input_arg("x").pipe( c.inline_expr("{cde} + 10").pass_args( cde=c.this().item("cde")))).pipe( c.inline_expr("{this} + {abc}").pass_args( this=c.this(), abc=c.label("abc"))), 2, ).gen_converter(debug=False) assert conv1(data_=1, x={"cde": 2}, y={"abc": 3}) == 15 list(c.generator_comp(c.this().add_label("a")).execute([1, 2])) c.list_comp(c.this().add_label("a")).execute([1, 2])
def test_iter_method(): assert (c.this().iter(c.this() * 3).filter(c.this()).as_type(list).execute( [1, 2, 3, 0, 1], debug=False, ) == [3, 6, 9, 3]) assert c.group_by(c.item(0)).aggregate( c([ c.item(0), c.item(1).pipe(c.ReduceFuncs.Max(c.this())), ]).iter(c.this() * 100).as_type(tuple)).execute([(0, 1), (0, 2), (1, 7)], debug=False) == [ (0, 200), (100, 700), ]
class A: x = 10 def __init__(self): self.x = 20 conv1 = (c.this() + c.input_arg("self").attr("x")).gen_converter(method=True) conv2 = (c.this + c.input_arg("cls").attr("x")).gen_converter(method=True) conv3 = classmethod( (c.this + c.input_arg("cls").attr("x")).gen_converter(class_method=True)) conv4 = classmethod( (c.this + c.input_arg("self").attr("x")).gen_converter(class_method=True)) conv5 = (c.this + c.input_arg("self").attr("x") + c.input_arg("n")).gen_converter( signature="self, n=1000, data_=15") conv6 = staticmethod( ((c.this + c.call_func(sum, c.input_arg("args"))) * c.input_arg("kwargs").call_method("get", "multiplicator", 1) ).gen_converter(signature="data_, *args, **kwargs"))
def test_list_comprehension(): assert c.list_comp(1).gen_converter()(range(5)) == [1] * 5 data = [{"name": "John"}, {"name": "Bill"}, {"name": "Nick"}] assert c.list_comp(c.item("name")).sort( key=lambda n: n).gen_converter()(data) == ["Bill", "John", "Nick"] assert c.list_comp(c.item("name")).sort().gen_converter()(data) == [ "Bill", "John", "Nick", ] assert tuple(c.generator_comp(c.item("name")).gen_converter()(data)) == ( "John", "Bill", "Nick", ) assert c.list_comp(c.item("name")).sort( key=lambda n: n, reverse=True).gen_converter()(data) == ["Nick", "John", "Bill"] assert c.list_comp({(c.item("name"), )}, ).execute(data) == [ {("John", )}, {("Bill", )}, {("Nick", )}, ] class CustomException(Exception): pass def f(): yield 1 raise CustomException wrapped_generator = c.generator_comp(c.this()).execute(f()) with pytest.raises(CustomException): list(wrapped_generator)
def test_slices(): assert c.this()[c.item(0):c.input_arg("slice_to"):c.item(1)].gen_converter( debug=False)([2, 2, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], slice_to=8) == [ 1, 3, 5, ]
def test_debug_true(): with c.OptionsCtx() as options: options.debug = True assert c.this().gen_converter(debug=True)(1) == 1 with pytest.raises(TypeError): assert c.item(0).gen_converter(debug=True)(1) == 1
def test_pipe_single_call_functions(): class CustomException(Exception): pass def one_off_func(): if one_off_func.first: one_off_func.first = False return 1 raise CustomException one_off_func.first = True assert (c.list_comp( c.call_func(one_off_func).pipe(( c.this() + 1, c.this() + 2, ))).gen_converter(debug=False)([1]) == [(2, 3)])
def test_pipes(): assert c.list_comp(c.inline_expr("{0} ** 2").pass_args(c.this())).pipe( c.call_func(sum, c.this())).pipe( c.call_func( lambda x, a: x + a, c.this(), c.naive({ "abc": 10 }).item(c.input_arg("key_name")), )).pipe([c.this(), c.this()]).execute([1, 2, 3], key_name="abc", debug=False) == [ 24, 24, ] assert c.item(0).pipe(datetime.strptime, "%Y-%m-%d").pipe( c.call_func(lambda dt: dt.date(), c.this())).execute(["2019-01-01"], debug=False) == date(2019, 1, 1) assert c.item(0).pipe(datetime.strptime, "%Y-%m-%d").pipe( c.this().call_method("date")).execute(["2019-01-01"], debug=False) == date(2019, 1, 1) conv = c.dict_comp( c.item("name"), c.item("transactions").pipe( c.list_comp({ "id": c.item(0).as_type(str), "amount": c.item(1).pipe(c.if_(c.this(), c.this().as_type(Decimal), None)), })), ).gen_converter(debug=False) assert conv([{ "name": "test", "transactions": [(0, 0), (1, 10)] }]) == { "test": [ { "id": "0", "amount": None }, { "id": "1", "amount": Decimal("10") }, ] }
def test_comprehension_filter_concats(): assert c.generator_comp(c.this()).filter(c.this() > 5).filter( c.this() < 10 ).as_type(list).execute(range(20), debug=False) == [6, 7, 8, 9] assert c.this().iter(c.this()).filter(c.this() > 5).filter( c.this() < 10 ).as_type(list).execute(range(20), debug=False) == [6, 7, 8, 9]
def test_pipes(): assert c.list_comp(c.inline_expr("{0} ** 2").pass_args(c.this())).pipe( c.call_func(sum, c.this())).pipe( c.call_func( lambda x, a: x + a, c.this(), c.naive({ "abc": 10 }).item(c.input_arg("key_name")), )).pipe([c.this(), c.this()]).execute([1, 2, 3], key_name="abc", debug=False) == [ 24, 24, ] assert c.item(0).pipe( datetime.strptime, "%Y-%m-%d", ).pipe(c.call_func(lambda dt: dt.date(), c.this())).execute([ "2019-01-01", ], debug=False) == date(2019, 1, 1) assert c.item(0).pipe( datetime.strptime, "%Y-%m-%d", ).pipe(c.this().call_method("date")).execute([ "2019-01-01", ], debug=False) == date(2019, 1, 1) with pytest.raises(c.ConversionException): c.naive(True).pipe(c.item("key1", _predefined_input={"key1": 777}))
def test_filter(): assert list(c.naive([1, 2, 3]).filter(c.this().gt(2)).execute(None)) == [3] assert c.filter(c.this().gt(1), cast=list).execute([1, 2, 3]) == [2, 3] assert c.filter(c.this().gt(1), cast=tuple).execute([1, 2, 3]) == (2, 3) assert c.filter(c.this().gt(1), cast=set).execute([1, 2, 3]) == {2, 3} assert c.filter(c.this().gt(1), cast=lambda x: list(x)).execute([1, 2, 3]) == [2, 3] assert c.list_comp(c.this()).filter(c.this().gt(1)).execute([1, 2, 3]) == [ 2, 3, ] assert c.this().filter(c.this().gt(1), cast=list).execute([1, 2, 3]) == [ 2, 3, ]
def test_join_with_complex_pipe(): def f(l): return l + [1, 3] pipeline = (c.aggregate(c.ReduceFuncs.Array(c.item("a"))).pipe( c.join(c.this(), c.call_func(f, c.this()), c.LEFT == c.RIGHT)).iter(c.item(1)).as_type(list)) assert (pipeline.execute([ { "a": 1 }, { "a": 2 }, { "a": 3 }, ]) == [1, 1, 2, 3, 3])
def test_reducer_inlining(dict_series): def f(): f.number_of_calls += 1 if f.number_of_calls > f.max_number_of_calls: raise Exception return [] f.max_number_of_calls = 1 f.number_of_calls = 0 converter = c.aggregate( c.ReduceFuncs.Array(c.item("name"), default=f, where=c.item("value") < 0).pipe( c.if_( if_true=c.this(), if_false=c.this(), ))).gen_converter(debug=False) assert converter(dict_series) == []
def test_naive_conversion_call(): assert c.naive("TEST").attr("lower").call().gen_converter()(100) == "test" assert c.call_func(str.lower, c.this()).gen_converter()("TEST") == "test" assert (c.naive("TE ST").attr("replace").call( " ", "").gen_converter()(100) == "TEST") f = MagicMock(return_value=1) c.naive(f).call(1, 2, test1=True, test2="test3").gen_converter()(100) f.assert_called_with(1, 2, test1=True, test2="test3") c.call(10, test="abc").gen_converter()(f) f.assert_called_with(10, test="abc")
def test_comprehension_where(): assert ( c.generator_comp(c.this().neg(), where=c.this() > 6) .as_type(list) .filter(c.this() > -9) .execute(range(10), debug=False) ) == [-7, -8] assert ( c.this() .iter(c.this().neg(), where=c.this() > 6) .as_type(list) .filter(c.this() > -9) .execute(range(10), debug=False) ) == [-7, -8] assert ( c.iter(c.this().neg(), where=c.this() > 6) .as_type(list) .filter(c.this() > -9) .execute(range(10), debug=False) ) == [-7, -8]
def test_mutation_item(): now = datetime.now() assert c.list_comp({ "name": c.item("fullName"), "age": c.item("age").as_type(int), "to_del": 1, }).pipe( c.list_comp( c.call_func(lambda d: d, c.this()).tap( c.Mut.set_item("name_before", c.label("_input").item(0, "name")), c.Mut.set_item("name", c.item("name").call_method("lower")), c.Mut.set_item("name_after", c.label("_input").item(0, "name")), c.Mut.set_item("_updated", c.input_arg("now")), c.Mut.set_item(c.item("age"), c.item("age") >= 18), c.Mut.del_item("to_del"), c.Mut.custom(c.this().call_method("update", {"to_add": 2})), )), label_input="_input", ).execute([{ "fullName": "John", "age": "28" }], debug=False, now=now) == [{ "name": "john", "name_after": "john", "name_before": "John", "age": 28, "_updated": now, 28: True, "to_add": 2, }] with pytest.raises(Exception): c.item(c.Mut.set_item("abc", "cde")) with pytest.raises(Exception): conversion = c.item(1) conversion.ensure_conversion(c.Mut.set_item("abc", "cde"), explicitly_allowed_cls=GetItem)
def test_pipe_conversion(): from convtools import conversion as c from convtools.base import PipeConversion assert PipeConversion(c.naive([1, 2, 3]), c.item(1)).execute(None) == 2 assert (PipeConversion(c.item("key1"), c.item("key2")).execute({"key1": { "key2": 3 }}, debug=False) == 3) assert (c.this().pipe(c.list_comp(c.this() + 1)).filter( c.this() > 3).execute([1, 2, 3, 4, 5, 6], debug=False)) == [4, 5, 6, 7] c.aggregate( c.ReduceFuncs.Array(c.item("key"), default=list).pipe( c.if_( c.call_func(any, c.generator_comp(c.this().is_(None))), c.call_func(list), c.this(), ))).gen_converter(debug=False)
def test_complex_labeling(): conv1 = (c.this().add_label("input").pipe( c.filter(c.this() % 3 == 0), label_input={ "input_type": c.call_func(type, c.this()) }, ).pipe( c.list_comp(c.this().as_type(str)), label_output={ "list_length": c.call_func(len, c.this()), "separator": c.if_(c.label("list_length") > 10, ",", ";"), }, ).pipe({ "result": c.label("separator").call_method("join", c.this()), "input_type": c.label("input_type"), "input_data": c.label("input"), }).gen_converter(debug=False)) assert conv1(range(30)) == { "result": "0;3;6;9;12;15;18;21;24;27", "input_type": range, "input_data": range(0, 30), } assert conv1(range(40)) == { "result": "0,3,6,9,12,15,18,21,24,27,30,33,36,39", "input_type": range, "input_data": range(0, 40), }