def etl():
return sequential(
csv_to_map([
'date', 'file', 'date2', 'log', 'app', 'beat', 'front', 'is_log',
'msg', 'offset', 'arch'
]), keep(["msg"]),
append(['msg'], wrap(lambda x: dict(Counter(x.lower())))))
def test_reader_csv_some_working():
operation = sequential(csv_to_map(["who"]), keep(fields=["who"]))
(transformed_output, transformed_error) = operation('"anciano"\r\n')
assert transformed_output is not None
assert "who" in transformed_output
assert transformed_output["who"] == "anciano"
assert transformed_error is None
def test_writer_json_some_working():
operation = sequential(keep(fields=["who"]), to_json)
(transformed_output, transformed_error) = operation({"who": "anciano"})
assert transformed_output is not None
assert transformed_output == '{"who": "anciano"}'
assert transformed_error is None
def test_emtpy():
op_p = parallel(None)
op_s = sequential(None)
(res_p, err_p) = op_p(None)
(res_s, err_s) = op_s(None)
assert res_p is None
assert res_s is None
assert err_p == err_s == 'No operations to perform'
def test_reader_csv_custom_params():
the_csv = '"Nori";"Melon"'
the_map = {"who": "Nori", "greet": "Melon"}
operation = sequential(csv_to_map(["who", "greet"], delimiter=";"), keep(fields=["who", "greet"]))
(transformed_output, transformed_error) = operation(the_csv)
assert transformed_output == the_map
assert transformed_error is None
reverse_operation = map_to_csv(["who", "greet"], delimiter=";")
(transformed_output, transformed_error) = reverse_operation(the_map)
assert transformed_output == the_csv
assert transformed_error is None
def test_two_operations_sequential():
def add_field(i, e):
i.update({"who": "Tom"})
return i, e
def change_field(i, e):
return "{} {}".format(i["greet"], i["who"]), e
op_p = sequential(add_field, change_field)
inp = {"greet": "hello"}
(res_p, err_p) = op_p(inp)
assert inp == {"greet": "hello"}
assert res_p == "hello Tom"
assert err_p is None
def test_one_opertaion():
def one_op(i, e=None):
i.update({"hello": "Tom"})
return i, e
op_p = parallel(one_op)
op_s = sequential(one_op)
inp = {"hello": "world"}
(res_p, err_p) = op_p(inp)
assert inp == {"hello": "world"}
(res_s, err_s) = op_s(inp)
assert res_p == {"hello": "Tom"}
assert res_s == {"hello": "Tom"}
assert err_p is None
assert err_s is None
def test_complex_workflow():
def adder(value):
def add_field(i, e):
i.update(value)
return i, e
return add_field
def change_field(i, e):
return {"hello": "Tom"}, e
complex_op = parallel(sequential(adder({"hello": "world"}), change_field),
adder({"world": False}))
inp = {"greet": "Hail"}
(res_c, err_c) = complex_op(inp)
assert inp == {"greet": "Hail"}
assert res_c == {'greet': 'Hail', 'hello': 'Tom', 'world': False}
assert err_c is None
def test_filtered_workflow():
def invert(x):
return not x
def must_true(x):
return x
def to_int(x):
if x:
return 1
return 0
inp = {"a": False}
op = sequential(substitution(["a"], wrap(invert)),
filter_tuple(["a"], wrap(must_true)),
substitution(["a"], wrap(to_int)))
(res, err) = op(inp)
assert res == {"a": 1}
assert err is None
def test_writer_csv_multiples_lines():
lines_in = [{
"who": "Thorin"
}, {
"who": "Dwalin"
}, {
"who": "Balin"
}, {
"who": "Kíli"
}, {
"who": "Fíli"
}, {
"who": "Dori"
}, {
"who": "Nori"
}, {
"who": "Ori"
}, {
"who": "Óin"
}, {
"who": "Glóin"
}, {
"who": "Bifur"
}, {
"who": "Bofur"
}, {
"who": "Bombur"
}]
lines_out_expected = [
'"Thorin"', '"Dwalin"', '"Balin"', '"Kíli"', '"Fíli"', '"Dori"',
'"Nori"', '"Ori"', '"Óin"', '"Glóin"', '"Bifur"', '"Bofur"', '"Bombur"'
]
operation = sequential(keep(fields=["who"]), map_to_csv(["who"]))
lines_out = [operation(x)[0] for x in lines_in]
assert lines_out_expected == lines_out
def test_writer_csv_some_working():
operation = sequential(keep(fields=["who"]), map_to_csv(["who"]))
(transformed_output, transformed_error) = operation({"who": "anciano"})
assert transformed_output is not None
assert transformed_output == '"anciano"'
assert transformed_error is None
def test_writer_csv_empty():
operation = sequential(keep(fields=["who"]), map_to_csv(["who"]))
(transformed_output, transformed_error) = operation(None)
assert transformed_output is None
assert transformed_error == {'who': 'who not found'}
def test_reader_csv_empty():
operation = sequential(csv_to_map(["who"]), keep(fields=["who"]))
(transformed_output, transformed_error) = operation(None)
assert transformed_output is None
assert transformed_error == {'who': 'no input'}
def test_reader_json_some_working():
operation = sequential(from_json, keep(fields=["who"]))
(transformed_output, transformed_error) = operation('{"who": "anciano"}')
assert "who" in transformed_output
assert transformed_output["who"] == "anciano"
assert transformed_error is None
def test_reader_json_empty():
operation = sequential(from_json, keep(fields=["who"]))
(transformed_output, transformed_error) = operation(None)
assert transformed_output is None
assert transformed_error == {'who': "Can't parse"}
def etl():
return sequential(
csv_to_map([
'date', 'file', 'date2', 'log', 'app', 'beat', 'front', 'is_log',
'msg', 'offset', 'arch'
]), keep(["msg"]))
