def test_list_comprehension():
assert c.list_comp(1).gen_converter()(range(5)) == [1] * 5
data = [{"name": "John"}, {"name": "Bill"}, {"name": "Nick"}]
assert c.list_comp(c.item("name")).sort(
key=lambda n: n).gen_converter()(data) == ["Bill", "John", "Nick"]
assert c.list_comp(c.item("name")).sort().gen_converter()(data) == [
"Bill",
"John",
"Nick",
]
assert tuple(c.generator_comp(c.item("name")).gen_converter()(data)) == (
"John",
"Bill",
"Nick",
)
assert c.list_comp(c.item("name")).sort(
key=lambda n: n,
reverse=True).gen_converter()(data) == ["Nick", "John", "Bill"]
assert c.list_comp({(c.item("name"), )}, ).execute(data) == [
{("John", )},
{("Bill", )},
{("Nick", )},
]
class CustomException(Exception):
pass
def f():
yield 1
raise CustomException
wrapped_generator = c.generator_comp(c.this()).execute(f())
with pytest.raises(CustomException):
list(wrapped_generator)
def test_comprehension_filter_cast_assumptions():
assert isinstance(
c.generator_comp(c.this).filter(c.this).execute(range(10)),
GeneratorType,
)
assert isinstance(
c.generator_comp(c.this).filter(c.this, cast=None).execute(range(10)),
GeneratorType,
)
assert (c.list_comp(c.this).filter(c.this).execute(range(3))) == [
1,
2,
]
def f(x):
f.number_of_calls += 1
if f.number_of_calls > f.max_number_of_calls:
raise ValueError
return bool(x)
f.max_number_of_calls = 2
f.number_of_calls = 0
assert (c.set_comp(c.this).filter(c.call_func(f,
c.this)).execute([0, 0,
1])) == {
1,
}
assert (c.set_comp(c.this).filter(c.this, cast=list).execute([0, 0,
1])) == [
1,
]
assert (c.set_comp(c.this).filter(c.this).execute(range(3))) == {
1,
2,
}
assert (c.tuple_comp(c.this).filter(c.this).execute(range(3))) == (
1,
2,
)
assert (c.tuple_comp(c.this).filter(c.this, list).execute(range(3))) == [
1,
2,
]
assert (c.dict_comp(c.this,
c.this).filter(c.item(0)).execute(range(3))) == {
1: 1,
2: 2,
}
assert (c.dict_comp(c.this, c.this).filter(c.item(0),
dict).execute(range(3))) == {
1: 1,
2: 2,
}
def test_generator_type_casts():
assert isinstance(c.generator_comp(c.this).as_type(list), c.list_comp)
assert isinstance(c.generator_comp(c.this).as_type(tuple), c.tuple_comp)
assert isinstance(c.generator_comp(c.this).as_type(set), c.set_comp)
conversion = c.this.iter(c.this).as_type(set)
assert isinstance(conversion, PipeConversion) and isinstance(
conversion.where, c.set_comp)
assert isinstance(
c.generator_comp(c.this).as_type(lambda x: list(x)), Call)
conversion = c.this.iter(c.this).as_type(lambda x: list(x))
assert isinstance(conversion, PipeConversion) and isinstance(
conversion.where, Call)
def test_list_comprehension():
assert c.list_comp(1).gen_converter()(range(5)) == [1] * 5
data = [{"name": "John"}, {"name": "Bill"}, {"name": "Nick"}]
assert c.list_comp(
c.item("name")).sort(key=lambda n: n).gen_converter()(data) == [
"Bill",
"John",
"Nick",
]
assert c.list_comp(c.item("name")).sort().gen_converter()(data) == [
"Bill",
"John",
"Nick",
]
assert tuple(c.generator_comp(c.item("name")).gen_converter()(data)) == (
"John",
"Bill",
"Nick",
)
assert c.list_comp(c.item("name")).sort(
key=lambda n: n,
reverse=True).gen_converter()(data) == ["Nick", "John", "Bill"]
assert c.list_comp({(c.item("name"), )}, ).execute(data) == [
{("John", )},
{("Bill", )},
{("Nick", )},
]
def test_comprehension_filter_concats():
assert c.generator_comp(c.this()).filter(c.this() > 5).filter(
c.this() < 10
).as_type(list).execute(range(20), debug=False) == [6, 7, 8, 9]
assert c.this().iter(c.this()).filter(c.this() > 5).filter(
c.this() < 10
).as_type(list).execute(range(20), debug=False) == [6, 7, 8, 9]
def test_generator_exception_handling():
class CustomException(Exception):
pass
def f_second_call_raises():
if f_second_call_raises.counter:
raise CustomException
f_second_call_raises.counter += 1
f_second_call_raises.counter = 0
conv = c.generator_comp(c.call_func(f_second_call_raises)).gen_converter()
with pytest.raises(CustomException):
list(conv([1, 2]))
def test_labels():
conv1 = c.if_(
1,
c.input_arg("y").item("abc").add_label("abc").pipe(
c.input_arg("x").pipe(
c.inline_expr("{cde} + 10").pass_args(
cde=c.this().item("cde")))).pipe(
c.inline_expr("{this} + {abc}").pass_args(
this=c.this(), abc=c.label("abc"))),
2,
).gen_converter(debug=False)
assert conv1(data_=1, x={"cde": 2}, y={"abc": 3}) == 15
list(c.generator_comp(c.this().add_label("a")).execute([1, 2]))
c.list_comp(c.this().add_label("a")).execute([1, 2])
def test_comprehensions_sorting():
assert c.generator_comp(c.this()).sort().execute(
[2, 1, 3], debug=False
) == [1, 2, 3]
assert c.list_comp(c.this()).sort().execute([2, 1, 3], debug=False) == [
1,
2,
3,
]
assert c.this().pipe(c.list_comp(c.this())).sort().execute(
[2, 1, 3], debug=False
) == [
1,
2,
3,
]
assert c.list_comp(c.this()).sort().sort(reverse=True).execute(
[2, 1, 3], debug=False
) == [3, 2, 1]
assert c.set_comp(c.this()).sort().execute([2, 2, 1, 3], debug=False) == [
1,
2,
3,
]
assert c.tuple_comp(c.this()).sort().execute(
[2, 2, 1, 3], debug=False
) == (
1,
2,
2,
3,
)
assert c.dict_comp(c.this() * -1, c.this()).sort().execute(
[2, 2, 1, 3], debug=False
) == OrderedDict(
[
(-3, 3),
(-2, 2),
(-1, 1),
]
)
def test_pipe_conversion():
from convtools import conversion as c
from convtools.base import PipeConversion
assert PipeConversion(c.naive([1, 2, 3]), c.item(1)).execute(None) == 2
assert (PipeConversion(c.item("key1"),
c.item("key2")).execute({"key1": {
"key2": 3
}},
debug=False) == 3)
assert (c.this.pipe(c.list_comp(c.this + 1)).filter(c.this > 3).execute(
[1, 2, 3, 4, 5, 6], debug=False)) == [4, 5, 6, 7]
c.aggregate(
c.ReduceFuncs.Array(c.item("key"), default=list).pipe(
c.if_(
c.call_func(any, c.generator_comp(c.this.is_(None))),
c.call_func(list),
c.this,
))).gen_converter(debug=False)
def test_comprehension_where():
assert (
c.generator_comp(c.this().neg(), where=c.this() > 6)
.as_type(list)
.filter(c.this() > -9)
.execute(range(10), debug=False)
) == [-7, -8]
assert (
c.this()
.iter(c.this().neg(), where=c.this() > 6)
.as_type(list)
.filter(c.this() > -9)
.execute(range(10), debug=False)
) == [-7, -8]
assert (
c.iter(c.this().neg(), where=c.this() > 6)
.as_type(list)
.filter(c.this() > -9)
.execute(range(10), debug=False)
) == [-7, -8]
def test_comprehension_where():
assert (c.generator_comp(
c.this.neg(),
where=c.this > 6).as_type(list).filter(c.this > -9).execute(
range(10), debug=False)) == [-7, -8]
assert (c.this.iter(c.this.neg(), where=c.this > 6).as_type(list).filter(
c.this > -9).execute(range(10), debug=False)) == [-7, -8]
assert (c.iter(c.this.neg(),
where=c.this > 6).as_type(list).filter(c.this > -9).execute(
range(10), debug=False)) == [-7, -8]
assert c.iter(c.this, where=c.and_(default=True)).as_type(list).execute(
range(3)) == [0, 1, 2]
assert c.iter(c.this, where=True).as_type(list).execute(range(3)) == [
0,
1,
2,
]
assert (c.iter(c.this, where=c.and_(default=False)).as_type(list).execute(
range(3)) == [])
def test_doc__index_deserialization():
class Employee:
def __init__(self, **kwargs):
self.kwargs = kwargs
input_data = {
"objects": [
{
"id": 1,
"first_name": "john",
"last_name": "black",
"dob": None,
"salary": "1,000.00",
"department": "D1 ",
"date": "2000-01-01",
},
{
"id": 2,
"first_name": "bob",
"last_name": "wick",
"dob": "1900-01-01",
"salary": "1,001.00",
"department": "D3 ",
"date": "2000-01-01",
},
]
}
# prepare a few conversions to reuse
c_strip = c.this.call_method("strip")
c_capitalize = c.this.call_method("capitalize")
c_decimal = c.this.call_method("replace", ",", "").as_type(Decimal)
c_date = c.call_func(datetime.strptime, c.this,
"%Y-%m-%d").call_method("date")
# reusing c_date
c_optional_date = c.if_(c.this, c_date, None)
first_name = c.item("first_name").pipe(c_capitalize)
last_name = c.item("last_name").pipe(c_capitalize)
# call "format" method of a string and pass first & last names as
# parameters
full_name = c("{} {}").call_method("format", first_name, last_name)
conv = (
c.item("objects").pipe(
c.generator_comp({
"id":
c.item("id"),
"first_name":
first_name,
"last_name":
last_name,
"full_name":
full_name,
"date_of_birth":
c.item("dob").pipe(c_optional_date),
"salary":
c.item("salary").pipe(c_decimal),
# pass a hardcoded dict and to get value by "department"
# key
"department_id":
c.naive({
"D1": 10,
"D2": 11,
"D3": 12,
}).item(c.item("department").pipe(c_strip)),
"date":
c.item("date").pipe(c_date),
})).pipe(
c.dict_comp(
c.item("id"), # key
c.apply_func( # value
Employee,
args=(),
kwargs=c.this,
),
)).gen_converter(debug=True) # to see print generated code
)
result = conv(input_data)
assert result[1].kwargs == {
"date": date(2000, 1, 1),
"date_of_birth": None,
"department_id": 10,
"first_name": "John",
"full_name": "John Black",
"id": 1,
"last_name": "Black",
"salary": Decimal("1000.00"),
}
assert result[2].kwargs == {
"date": date(2000, 1, 1),
"date_of_birth": date(1900, 1, 1),
"department_id": 12,
"first_name": "Bob",
"full_name": "Bob Wick",
"id": 2,
"last_name": "Wick",
"salary": Decimal("1001.00"),
}
def test_doc__index_deserialization():
class Employee:
def __init__(self, **kwargs):
self.kwargs = kwargs
input_data = {
"objects": [
{
"id": 1,
"first_name": "john",
"last_name": "black",
"dob": None,
"salary": "1,000.00",
"department": "D1 ",
"date": "2000-01-01",
},
{
"id": 2,
"first_name": "bob",
"last_name": "wick",
"dob": "1900-01-01",
"salary": "1,001.00",
"department": "D3 ",
"date": "2000-01-01",
},
]
}
# get by "department" key and then call method "strip"
department = c.item("department").call_method("strip")
first_name = c.item("first_name").call_method("capitalize")
last_name = c.item("last_name").call_method("capitalize")
# call "format" method of a string and pass first & last names as
# parameters
full_name = c("{} {}").call_method("format", first_name, last_name)
date_of_birth = c.item("dob")
# partially initialized "strptime"
parse_date = c.call_func(datetime.strptime, c.this(),
"%Y-%m-%d").call_method("date")
conv = (
c.item("objects").pipe(
c.generator_comp({
"id":
c.item("id"),
"first_name":
first_name,
"last_name":
last_name,
"full_name":
full_name,
"date_of_birth":
c.if_(
date_of_birth,
date_of_birth.pipe(parse_date),
None,
),
"salary":
c.call_func(
Decimal,
c.item("salary").call_method("replace", ",", ""),
),
# pass a hardcoded dict and to get value by "department"
# key
"department_id":
c.naive({
"D1": 10,
"D2": 11,
"D3": 12,
}).item(department),
"date":
c.item("date").pipe(parse_date),
})).
pipe(
c.dict_comp(
c.item(
"id"), # key
# write a python code expression, format with passed parameters
c.inline_expr("{employee_cls}(**{kwargs})").pass_args(
employee_cls=Employee,
kwargs=c.this(),
), # value
)).gen_converter(debug=True))
result = conv(input_data)
assert result[1].kwargs == {
"date": date(2000, 1, 1),
"date_of_birth": None,
"department_id": 10,
"first_name": "John",
"full_name": "John Black",
"id": 1,
"last_name": "Black",
"salary": Decimal("1000.00"),
}
assert result[2].kwargs == {
"date": date(2000, 1, 1),
"date_of_birth": date(1900, 1, 1),
"department_id": 12,
"first_name": "Bob",
"full_name": "Bob Wick",
"id": 2,
"last_name": "Wick",
"salary": Decimal("1001.00"),
}
def test_doc__index_word_count():
# Let's say we need to count words across all files
input_data = [
"war-and-peace-1.txt",
"war-and-peace-2.txt",
"war-and-peace-3.txt",
"war-and-peace-4.txt",
]
# # iterate an input and read file lines
#
# def read_file(filename):
# with open(filename) as f:
# for line in f:
# yield line
# extract_strings = c.generator_comp(c.call_func(read_file, c.this()))
# to simplify testing
extract_strings = c.generator_comp(
c.call_func(lambda filename: [filename], c.this()))
# 1. make ``re`` pattern available to the code to be generated
# 2. call ``finditer`` method of the pattern and pass the string
# as an argument
# 3. pass the result to the next conversion
# 4. iterate results, call ``.group()`` method of each re.Match
# and call ``.lower()`` on each result
split_words = (c.naive(re.compile(r"\w+")).call_method(
"finditer", c.this()).pipe(
c.generator_comp(c.this().call_method("group",
0).call_method("lower"))))
# ``extract_strings`` is the generator of strings
# so we iterate it and pass each item to ``split_words`` conversion
vectorized_split_words = c.generator_comp(c.this().pipe(split_words))
# flattening the result of ``vectorized_split_words``, which is
# a generator of generators of strings
flatten = c.call_func(
chain.from_iterable,
c.this(),
)
# aggregate the input, the result is a single dict
# words are keys, values are count of words
dict_word_to_count = c.aggregate(
c.ReduceFuncs.DictCount(c.this(), c.this(), default=dict))
# take top N words by:
# - call ``.items()`` method of the dict (the result of the aggregate)
# - pass the result to ``sorted``
# - take the slice, using input argument named ``top_n``
# - cast to a dict
take_top_n = (c.this().call_method("items").sort(
key=lambda t: t[1],
reverse=True).pipe(c.this()[:c.input_arg("top_n")]).as_type(dict))
# the resulting pipeline is pretty self-descriptive, except the ``c.if_``
# part, which checks the condition (first argument),
# and returns the 2nd if True OR the 3rd (input data by default) otherwise
pipeline = (
extract_strings.pipe(flatten).pipe(vectorized_split_words).pipe(
flatten).pipe(dict_word_to_count).pipe(
c.if_(
c.input_arg("top_n").is_not(None),
c.this().pipe(take_top_n),
))
# Define the resulting converter function signature. In fact this
# isn't necessary if you don't need to specify default values
).gen_converter(debug=True, signature="data_, top_n=None")
assert pipeline(input_data, top_n=3) == {"war": 4, "and": 4, "peace": 4}
def test_pipes():
assert c.list_comp(c.inline_expr("{0} ** 2").pass_args(c.this())).pipe(
c.call_func(sum, c.this())).pipe(
c.call_func(
lambda x, a: x + a,
c.this(),
c.naive({
"abc": 10
}).item(c.input_arg("key_name")),
)).pipe([c.this(), c.this()]).execute([1, 2, 3],
key_name="abc",
debug=False) == [
24,
24,
]
assert c.item(0).pipe(datetime.strptime, "%Y-%m-%d").pipe(
c.call_func(lambda dt: dt.date(),
c.this())).execute(["2019-01-01"],
debug=False) == date(2019, 1, 1)
assert c.item(0).pipe(datetime.strptime, "%Y-%m-%d").pipe(
c.this().call_method("date")).execute(["2019-01-01"],
debug=False) == date(2019, 1, 1)
with c.OptionsCtx() as options:
max_pipe_length = options.max_pipe_length = 10
with pytest.raises(c.ConversionException):
conv = c.this()
for i in range(max_pipe_length + 1):
conv = c.this().pipe(conv)
with c.OptionsCtx() as options2, pytest.raises(c.ConversionException):
options2.max_pipe_length = 5
conv.clone()
conv = c.dict_comp(
c.item("name"),
c.item("transactions").pipe(
c.list_comp({
"id":
c.item(0).as_type(str),
"amount":
c.item(1).pipe(c.if_(c.this(),
c.this().as_type(Decimal), None)),
})),
).gen_converter(debug=True)
assert conv([{
"name": "test",
"transactions": [(0, 0), (1, 10)]
}]) == {
"test": [
{
"id": "0",
"amount": None
},
{
"id": "1",
"amount": Decimal("10")
},
]
}
with c.OptionsCtx() as options:
max_pipe_length = options.max_pipe_length = 10
conv1 = c.item(0).pipe(c.item(1).pipe(c.item(2)))
def measure_pipe_length(conv):
length = 0
for i in range(max_pipe_length):
if conv._predefined_input is not None:
length += 1
conv = conv._predefined_input
else:
break
return length
pipe_length_before = measure_pipe_length(conv1)
for i in range(max_pipe_length + 20):
c.generator_comp(c.this().pipe(conv1))
pipe_length_after = measure_pipe_length(conv1)
assert pipe_length_after == pipe_length_before
