def test_pipe_filter_sort():
assert (c.this().as_type(list).pipe(c.iter(c.this() + 1)).filter(
c.this() > 3).sort(key=lambda x: x,
reverse=True).execute(range(7),
debug=False)) == [7, 6, 5, 4]
assert c.this().sort().execute([3, 1, 2]) == [1, 2, 3]
def test_linecache_cleaning():
_ConverterCallable.linecache_keys.max_size = 100
length_before = len(linecache.cache)
for i in range(100):
c.this().gen_converter(debug=False)
length_after_100 = len(linecache.cache)
for i in range(10):
c.this().gen_converter(debug=False)
length_after_110 = len(linecache.cache)
assert (length_after_110 == length_after_100
and length_before + 100 >= length_after_100)
for key in list(linecache.cache.keys()):
del linecache.cache[key]
converter_callable = c.this().gen_converter(debug=False)
for (
fake_filename,
code_str,
) in converter_callable._fake_filename_to_code_str.items():
converter_callable.add_sources(fake_filename, code_str)
with pytest.raises(Exception):
converter_callable.add_sources(fake_filename, code_str + " ")
def test_memory_freeing():
converter = (
c.this()
.pipe(
c.list_comp(c.this() + c.label("input_data").item(0)),
label_input=dict(input_data=c.this()),
)
.gen_converter(debug=True)
)
sizes = []
sizes.append(total_size(converter.__dict__))
for i in range(100):
l_input = [i + j for j in range(3)]
l_out = [j + l_input[0] for j in l_input]
assert converter(l_input) == l_out
sizes.append(total_size(converter.__dict__))
assert all(sizes[0] == size for size in sizes[1:]), sizes
conv2 = (
c.inline_expr("globals().__setitem__('a', {}) or 1")
.pass_args(c.this())
.gen_converter()
)
with pytest.raises(AssertionError):
# should raise because of a memory leak
conv2(123)
def test_group_by_reducer_clones():
data = [
{
"value": 2
},
{
"value": 3
},
]
conv = c.aggregate(
c.item("value").pipe(c.ReduceFuncs.Sum(c.this()).pipe(c.this() + 1)))
assert conv.execute(data) == 6
reducer = c.ReduceFuncs.DictSum(c.item("k"), c.item("v"))
reducer1 = c.item("item1").pipe(reducer)
reducer2 = c.item("item2").pipe(reducer)
assert c.aggregate(reducer1).execute([{
"item1": {
"k": 1,
"v": 2
}
}]) == {
1: 2
}
assert c.aggregate(reducer2).execute([{
"item2": {
"k": 2,
"v": 3
}
}]) == {
2: 3
}
def test_group_by_with_double_ended_pipes():
input_data = [
{
"value": 1
},
{
"value": 2
},
]
# fmt: off
conv = c.aggregate(
c.item("value").pipe(c.ReduceFuncs.Sum(c.this())).pipe(
c.this() * 2)).gen_converter()
# fmt: on
result = conv(input_data)
assert result == 6
input_data = [
{
"k": "A",
"v": 1
},
{
"k": "A",
"v": 2
},
]
reducer = c.ReduceFuncs.Sum(c.item("v"))
conv = (c.group_by(c.item("k")).aggregate({
"v1":
c.input_arg("test").pipe(reducer),
"v2":
reducer,
}).gen_converter())
assert conv(input_data, test={"v": 7}) == [{"v1": 14, "v2": 3}]
def test_sort():
assert c.sort().execute([2, 3, 1]) == [1, 2, 3]
assert c.sort(key=lambda x: x, reverse=True).execute([2, 3, 1]) == [
3,
2,
1,
]
assert c.this().sort().execute([2, 3, 1]) == [1, 2, 3]
assert c.this().sort(key=lambda x: x,
reverse=False).execute([2, 3, 1]) == [1, 2, 3]
def test_simple_label():
conv1 = (c.tuple(c.item(2).add_label("a"), c.this()).pipe(
c.item(1).pipe(c.list_comp(
(c.this(), c.label("a"))))).gen_converter(debug=False))
assert conv1([1, 2, 3, 4]) == [(1, 3), (2, 3), (3, 3), (4, 3)]
conv2 = (c.tuple(c.item(1).add_label("a"), c.this()).pipe(
c.item(1),
label_input={
"aa": c.item(0),
"bb": c.item(0)
},
label_output="collection1",
).pipe(
c.label("collection1").pipe(
c.aggregate(
c.ReduceFuncs.Sum(
c.this() + c.label("a") + c.label("aa") +
c.input_arg("x") + c.label("collection1").item(0), ))),
label_output="b",
).pipe(c.this() + c.label("b")).gen_converter(debug=False))
assert conv2([1, 2, 3, 4], x=10) == 140
conv3 = (c.tuple(c.item("default").add_label("default"), c.this()).pipe(
c.item(1).pipe(c.item(
"abc", default=c.label("default")))).gen_converter(debug=False))
assert conv3({"default": 1}) == 1
with pytest.raises(c.ConversionException):
c.this().pipe(c.this(), label_input=1)
def test_multiple_aggregations(dict_series):
assert (
c.aggregate(c.ReduceFuncs.Array(c.item("name")))
.pipe(
c.aggregate(c.ReduceFuncs.ArrayDistinct(c.this())).pipe(
c.aggregate(c.ReduceFuncs.Max(c.this()))
)
)
.execute(dict_series, debug=False)
== "Nick"
)
def test_optional_dict():
conv = c.list_comp({
"key1":
c.item("key1"),
"key2":
c.optional(c.item("key2", default=None)),
"key3":
c.optional(c.item("key1") * 200, skip_value=2000),
"key4":
c.optional(
c.item("key1") * c.input_arg("x") * 300,
skip_if=c.item("key1") < 5,
),
"key5":
c.optional(
c.item("key1") * c.input_arg("x") * 300,
keep_if=c.item("key1") >= 5,
),
c.optional(c.item("key2", default=-1), skip_value=-1):
0,
c.optional(c.item("key1") * 400, skip_if=c.item("key1") < 5):
c.optional(c.item("key22")),
c.optional(c.item("key1") * 500, skip_if=c.item("key1") < 5):
c.optional(c.item("key22"), skip_value=20),
}).gen_converter(debug=False)
assert conv([{
"key1": 1,
"key2": 2
}, {
"key1": 10,
"key22": 20
}], x=1) == [
{
"key1": 1,
"key2": 2,
"key3": 200,
2: 0
},
{
"key1": 10,
"key4": 3000,
"key5": 3000,
4000: 20
},
]
with pytest.raises(Exception):
c.list_comp(c.optional(c.item("key1"))).gen_converter()
with pytest.raises(Exception):
c.optional(c.item("key1"), skip_value=1, skip_if=c.this())
with pytest.raises(Exception):
c.this().pipe(c.optional(c.this()))
def test_min_max():
assert c.min(0, 1).execute(None) == 0
assert c.min(2, 1).execute(None) == 1
assert c.max(0, 1).execute(None) == 1
assert c.max(2, 1).execute(None) == 2
assert c.min(c.item(0), c.item(1)).execute((0, 1)) == 0
assert c((2, 1)).pipe(c.min(c.item(0), c.item(1))).execute(None) == 1
with pytest.raises(TypeError):
c.min(c.this()).execute(-1)
with pytest.raises(TypeError):
c.max(c.this()).execute(-1)
def test_legacy_dict_reduce_approach(dict_series):
output = c.aggregate(
c.reduce(
c.ReduceFuncs.DictSum,
(c.item("name"), c.item("value")),
)).execute(dict_series)
assert output == {
"Nick": 3,
"John": 63,
}
with pytest.raises(ValueError):
c.ReduceFuncs.DictSum(c.this(), c.this(), c.this())
with pytest.raises(ValueError):
c.ReduceFuncs.DictSum({c.this(), c.this()})
def test_caching_conversion():
class CustomException(Exception):
pass
def f(number):
if not f.first_time:
raise CustomException
f.first_time = False
return number
f.first_time = True
conv = (c.call_func(f, c.this()).pipe(
c.if_(c.this(),
c.this() + 1,
c.this() + 2)).gen_converter())
assert conv(0) == 2
with pytest.raises(CustomException):
assert conv(0) == 2
f.first_time = True
assert conv(1) == 2
with pytest.raises(CustomException):
c.call_func(f, c.this()).pipe(
c.if_(c.this(), c.this() + 1, c.this() + 2,
no_input_caching=True)).execute(0)
def test_labels():
conv1 = c.if_(
1,
c.input_arg("y").item("abc").add_label("abc").pipe(
c.input_arg("x").pipe(
c.inline_expr("{cde} + 10").pass_args(
cde=c.this().item("cde")))).pipe(
c.inline_expr("{this} + {abc}").pass_args(
this=c.this(), abc=c.label("abc"))),
2,
).gen_converter(debug=False)
assert conv1(data_=1, x={"cde": 2}, y={"abc": 3}) == 15
list(c.generator_comp(c.this().add_label("a")).execute([1, 2]))
c.list_comp(c.this().add_label("a")).execute([1, 2])
def test_iter_method():
assert (c.this().iter(c.this() * 3).filter(c.this()).as_type(list).execute(
[1, 2, 3, 0, 1],
debug=False,
) == [3, 6, 9, 3])
assert c.group_by(c.item(0)).aggregate(
c([
c.item(0),
c.item(1).pipe(c.ReduceFuncs.Max(c.this())),
]).iter(c.this() * 100).as_type(tuple)).execute([(0, 1), (0, 2),
(1, 7)],
debug=False) == [
(0, 200),
(100, 700),
]
class A:
x = 10
def __init__(self):
self.x = 20
conv1 = (c.this() +
c.input_arg("self").attr("x")).gen_converter(method=True)
conv2 = (c.this +
c.input_arg("cls").attr("x")).gen_converter(method=True)
conv3 = classmethod(
(c.this +
c.input_arg("cls").attr("x")).gen_converter(class_method=True))
conv4 = classmethod(
(c.this +
c.input_arg("self").attr("x")).gen_converter(class_method=True))
conv5 = (c.this + c.input_arg("self").attr("x") +
c.input_arg("n")).gen_converter(
signature="self, n=1000, data_=15")
conv6 = staticmethod(
((c.this + c.call_func(sum, c.input_arg("args"))) *
c.input_arg("kwargs").call_method("get", "multiplicator", 1)
).gen_converter(signature="data_, *args, **kwargs"))
def test_list_comprehension():
assert c.list_comp(1).gen_converter()(range(5)) == [1] * 5
data = [{"name": "John"}, {"name": "Bill"}, {"name": "Nick"}]
assert c.list_comp(c.item("name")).sort(
key=lambda n: n).gen_converter()(data) == ["Bill", "John", "Nick"]
assert c.list_comp(c.item("name")).sort().gen_converter()(data) == [
"Bill",
"John",
"Nick",
]
assert tuple(c.generator_comp(c.item("name")).gen_converter()(data)) == (
"John",
"Bill",
"Nick",
)
assert c.list_comp(c.item("name")).sort(
key=lambda n: n,
reverse=True).gen_converter()(data) == ["Nick", "John", "Bill"]
assert c.list_comp({(c.item("name"), )}, ).execute(data) == [
{("John", )},
{("Bill", )},
{("Nick", )},
]
class CustomException(Exception):
pass
def f():
yield 1
raise CustomException
wrapped_generator = c.generator_comp(c.this()).execute(f())
with pytest.raises(CustomException):
list(wrapped_generator)
def test_slices():
assert c.this()[c.item(0):c.input_arg("slice_to"):c.item(1)].gen_converter(
debug=False)([2, 2, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], slice_to=8) == [
1,
3,
5,
]
def test_debug_true():
with c.OptionsCtx() as options:
options.debug = True
assert c.this().gen_converter(debug=True)(1) == 1
with pytest.raises(TypeError):
assert c.item(0).gen_converter(debug=True)(1) == 1
def test_pipe_single_call_functions():
class CustomException(Exception):
pass
def one_off_func():
if one_off_func.first:
one_off_func.first = False
return 1
raise CustomException
one_off_func.first = True
assert (c.list_comp(
c.call_func(one_off_func).pipe((
c.this() + 1,
c.this() + 2,
))).gen_converter(debug=False)([1]) == [(2, 3)])
def test_pipes():
assert c.list_comp(c.inline_expr("{0} ** 2").pass_args(c.this())).pipe(
c.call_func(sum, c.this())).pipe(
c.call_func(
lambda x, a: x + a,
c.this(),
c.naive({
"abc": 10
}).item(c.input_arg("key_name")),
)).pipe([c.this(), c.this()]).execute([1, 2, 3],
key_name="abc",
debug=False) == [
24,
24,
]
assert c.item(0).pipe(datetime.strptime, "%Y-%m-%d").pipe(
c.call_func(lambda dt: dt.date(),
c.this())).execute(["2019-01-01"],
debug=False) == date(2019, 1, 1)
assert c.item(0).pipe(datetime.strptime, "%Y-%m-%d").pipe(
c.this().call_method("date")).execute(["2019-01-01"],
debug=False) == date(2019, 1, 1)
conv = c.dict_comp(
c.item("name"),
c.item("transactions").pipe(
c.list_comp({
"id":
c.item(0).as_type(str),
"amount":
c.item(1).pipe(c.if_(c.this(),
c.this().as_type(Decimal), None)),
})),
).gen_converter(debug=False)
assert conv([{
"name": "test",
"transactions": [(0, 0), (1, 10)]
}]) == {
"test": [
{
"id": "0",
"amount": None
},
{
"id": "1",
"amount": Decimal("10")
},
]
}
def test_comprehension_filter_concats():
assert c.generator_comp(c.this()).filter(c.this() > 5).filter(
c.this() < 10
).as_type(list).execute(range(20), debug=False) == [6, 7, 8, 9]
assert c.this().iter(c.this()).filter(c.this() > 5).filter(
c.this() < 10
).as_type(list).execute(range(20), debug=False) == [6, 7, 8, 9]
def test_pipes():
assert c.list_comp(c.inline_expr("{0} ** 2").pass_args(c.this())).pipe(
c.call_func(sum, c.this())).pipe(
c.call_func(
lambda x, a: x + a,
c.this(),
c.naive({
"abc": 10
}).item(c.input_arg("key_name")),
)).pipe([c.this(), c.this()]).execute([1, 2, 3],
key_name="abc",
debug=False) == [
24,
24,
]
assert c.item(0).pipe(
datetime.strptime,
"%Y-%m-%d",
).pipe(c.call_func(lambda dt: dt.date(),
c.this())).execute([
"2019-01-01",
], debug=False) == date(2019, 1, 1)
assert c.item(0).pipe(
datetime.strptime,
"%Y-%m-%d",
).pipe(c.this().call_method("date")).execute([
"2019-01-01",
], debug=False) == date(2019, 1, 1)
with pytest.raises(c.ConversionException):
c.naive(True).pipe(c.item("key1", _predefined_input={"key1": 777}))
def test_filter():
assert list(c.naive([1, 2, 3]).filter(c.this().gt(2)).execute(None)) == [3]
assert c.filter(c.this().gt(1), cast=list).execute([1, 2, 3]) == [2, 3]
assert c.filter(c.this().gt(1), cast=tuple).execute([1, 2, 3]) == (2, 3)
assert c.filter(c.this().gt(1), cast=set).execute([1, 2, 3]) == {2, 3}
assert c.filter(c.this().gt(1),
cast=lambda x: list(x)).execute([1, 2, 3]) == [2, 3]
assert c.list_comp(c.this()).filter(c.this().gt(1)).execute([1, 2, 3]) == [
2,
3,
]
assert c.this().filter(c.this().gt(1), cast=list).execute([1, 2, 3]) == [
2,
3,
]
def test_join_with_complex_pipe():
def f(l):
return l + [1, 3]
pipeline = (c.aggregate(c.ReduceFuncs.Array(c.item("a"))).pipe(
c.join(c.this(), c.call_func(f, c.this()),
c.LEFT == c.RIGHT)).iter(c.item(1)).as_type(list))
assert (pipeline.execute([
{
"a": 1
},
{
"a": 2
},
{
"a": 3
},
]) == [1, 1, 2, 3, 3])
def test_reducer_inlining(dict_series):
def f():
f.number_of_calls += 1
if f.number_of_calls > f.max_number_of_calls:
raise Exception
return []
f.max_number_of_calls = 1
f.number_of_calls = 0
converter = c.aggregate(
c.ReduceFuncs.Array(c.item("name"),
default=f,
where=c.item("value") < 0).pipe(
c.if_(
if_true=c.this(),
if_false=c.this(),
))).gen_converter(debug=False)
assert converter(dict_series) == []
def test_naive_conversion_call():
assert c.naive("TEST").attr("lower").call().gen_converter()(100) == "test"
assert c.call_func(str.lower, c.this()).gen_converter()("TEST") == "test"
assert (c.naive("TE ST").attr("replace").call(
" ", "").gen_converter()(100) == "TEST")
f = MagicMock(return_value=1)
c.naive(f).call(1, 2, test1=True, test2="test3").gen_converter()(100)
f.assert_called_with(1, 2, test1=True, test2="test3")
c.call(10, test="abc").gen_converter()(f)
f.assert_called_with(10, test="abc")
def test_comprehension_where():
assert (
c.generator_comp(c.this().neg(), where=c.this() > 6)
.as_type(list)
.filter(c.this() > -9)
.execute(range(10), debug=False)
) == [-7, -8]
assert (
c.this()
.iter(c.this().neg(), where=c.this() > 6)
.as_type(list)
.filter(c.this() > -9)
.execute(range(10), debug=False)
) == [-7, -8]
assert (
c.iter(c.this().neg(), where=c.this() > 6)
.as_type(list)
.filter(c.this() > -9)
.execute(range(10), debug=False)
) == [-7, -8]
def test_mutation_item():
now = datetime.now()
assert c.list_comp({
"name": c.item("fullName"),
"age": c.item("age").as_type(int),
"to_del": 1,
}).pipe(
c.list_comp(
c.call_func(lambda d: d, c.this()).tap(
c.Mut.set_item("name_before",
c.label("_input").item(0, "name")),
c.Mut.set_item("name",
c.item("name").call_method("lower")),
c.Mut.set_item("name_after",
c.label("_input").item(0, "name")),
c.Mut.set_item("_updated", c.input_arg("now")),
c.Mut.set_item(c.item("age"),
c.item("age") >= 18),
c.Mut.del_item("to_del"),
c.Mut.custom(c.this().call_method("update", {"to_add": 2})),
)),
label_input="_input",
).execute([{
"fullName": "John",
"age": "28"
}], debug=False, now=now) == [{
"name": "john",
"name_after": "john",
"name_before": "John",
"age": 28,
"_updated": now,
28: True,
"to_add": 2,
}]
with pytest.raises(Exception):
c.item(c.Mut.set_item("abc", "cde"))
with pytest.raises(Exception):
conversion = c.item(1)
conversion.ensure_conversion(c.Mut.set_item("abc", "cde"),
explicitly_allowed_cls=GetItem)
def test_pipe_conversion():
from convtools import conversion as c
from convtools.base import PipeConversion
assert PipeConversion(c.naive([1, 2, 3]), c.item(1)).execute(None) == 2
assert (PipeConversion(c.item("key1"),
c.item("key2")).execute({"key1": {
"key2": 3
}},
debug=False) == 3)
assert (c.this().pipe(c.list_comp(c.this() + 1)).filter(
c.this() > 3).execute([1, 2, 3, 4, 5, 6],
debug=False)) == [4, 5, 6, 7]
c.aggregate(
c.ReduceFuncs.Array(c.item("key"), default=list).pipe(
c.if_(
c.call_func(any, c.generator_comp(c.this().is_(None))),
c.call_func(list),
c.this(),
))).gen_converter(debug=False)
def test_complex_labeling():
conv1 = (c.this().add_label("input").pipe(
c.filter(c.this() % 3 == 0),
label_input={
"input_type": c.call_func(type, c.this())
},
).pipe(
c.list_comp(c.this().as_type(str)),
label_output={
"list_length": c.call_func(len, c.this()),
"separator": c.if_(c.label("list_length") > 10, ",", ";"),
},
).pipe({
"result": c.label("separator").call_method("join", c.this()),
"input_type": c.label("input_type"),
"input_data": c.label("input"),
}).gen_converter(debug=False))
assert conv1(range(30)) == {
"result": "0;3;6;9;12;15;18;21;24;27",
"input_type": range,
"input_data": range(0, 30),
}
assert conv1(range(40)) == {
"result": "0,3,6,9,12,15,18,21,24,27,30,33,36,39",
"input_type": range,
"input_data": range(0, 40),
}
