def etl(): return sequential( csv_to_map([ 'date', 'file', 'date2', 'log', 'app', 'beat', 'front', 'is_log', 'msg', 'offset', 'arch' ]), keep(["msg"]), append(['msg'], wrap(lambda x: dict(Counter(x.lower())))))
def test_reader_csv_some_working(): operation = sequential(csv_to_map(["who"]), keep(fields=["who"])) (transformed_output, transformed_error) = operation('"anciano"\r\n') assert transformed_output is not None assert "who" in transformed_output assert transformed_output["who"] == "anciano" assert transformed_error is None
def test_writer_json_some_working(): operation = sequential(keep(fields=["who"]), to_json) (transformed_output, transformed_error) = operation({"who": "anciano"}) assert transformed_output is not None assert transformed_output == '{"who": "anciano"}' assert transformed_error is None
def test_emtpy(): op_p = parallel(None) op_s = sequential(None) (res_p, err_p) = op_p(None) (res_s, err_s) = op_s(None) assert res_p is None assert res_s is None assert err_p == err_s == 'No operations to perform'
def test_reader_csv_custom_params(): the_csv = '"Nori";"Melon"' the_map = {"who": "Nori", "greet": "Melon"} operation = sequential(csv_to_map(["who", "greet"], delimiter=";"), keep(fields=["who", "greet"])) (transformed_output, transformed_error) = operation(the_csv) assert transformed_output == the_map assert transformed_error is None reverse_operation = map_to_csv(["who", "greet"], delimiter=";") (transformed_output, transformed_error) = reverse_operation(the_map) assert transformed_output == the_csv assert transformed_error is None
def test_two_operations_sequential(): def add_field(i, e): i.update({"who": "Tom"}) return i, e def change_field(i, e): return "{} {}".format(i["greet"], i["who"]), e op_p = sequential(add_field, change_field) inp = {"greet": "hello"} (res_p, err_p) = op_p(inp) assert inp == {"greet": "hello"} assert res_p == "hello Tom" assert err_p is None
def test_one_opertaion(): def one_op(i, e=None): i.update({"hello": "Tom"}) return i, e op_p = parallel(one_op) op_s = sequential(one_op) inp = {"hello": "world"} (res_p, err_p) = op_p(inp) assert inp == {"hello": "world"} (res_s, err_s) = op_s(inp) assert res_p == {"hello": "Tom"} assert res_s == {"hello": "Tom"} assert err_p is None assert err_s is None
def test_complex_workflow(): def adder(value): def add_field(i, e): i.update(value) return i, e return add_field def change_field(i, e): return {"hello": "Tom"}, e complex_op = parallel(sequential(adder({"hello": "world"}), change_field), adder({"world": False})) inp = {"greet": "Hail"} (res_c, err_c) = complex_op(inp) assert inp == {"greet": "Hail"} assert res_c == {'greet': 'Hail', 'hello': 'Tom', 'world': False} assert err_c is None
def test_filtered_workflow(): def invert(x): return not x def must_true(x): return x def to_int(x): if x: return 1 return 0 inp = {"a": False} op = sequential(substitution(["a"], wrap(invert)), filter_tuple(["a"], wrap(must_true)), substitution(["a"], wrap(to_int))) (res, err) = op(inp) assert res == {"a": 1} assert err is None
def test_writer_csv_multiples_lines(): lines_in = [{ "who": "Thorin" }, { "who": "Dwalin" }, { "who": "Balin" }, { "who": "Kíli" }, { "who": "Fíli" }, { "who": "Dori" }, { "who": "Nori" }, { "who": "Ori" }, { "who": "Óin" }, { "who": "Glóin" }, { "who": "Bifur" }, { "who": "Bofur" }, { "who": "Bombur" }] lines_out_expected = [ '"Thorin"', '"Dwalin"', '"Balin"', '"Kíli"', '"Fíli"', '"Dori"', '"Nori"', '"Ori"', '"Óin"', '"Glóin"', '"Bifur"', '"Bofur"', '"Bombur"' ] operation = sequential(keep(fields=["who"]), map_to_csv(["who"])) lines_out = [operation(x)[0] for x in lines_in] assert lines_out_expected == lines_out
def test_writer_csv_some_working(): operation = sequential(keep(fields=["who"]), map_to_csv(["who"])) (transformed_output, transformed_error) = operation({"who": "anciano"}) assert transformed_output is not None assert transformed_output == '"anciano"' assert transformed_error is None
def test_writer_csv_empty(): operation = sequential(keep(fields=["who"]), map_to_csv(["who"])) (transformed_output, transformed_error) = operation(None) assert transformed_output is None assert transformed_error == {'who': 'who not found'}
def test_reader_csv_empty(): operation = sequential(csv_to_map(["who"]), keep(fields=["who"])) (transformed_output, transformed_error) = operation(None) assert transformed_output is None assert transformed_error == {'who': 'no input'}
def test_reader_json_some_working(): operation = sequential(from_json, keep(fields=["who"])) (transformed_output, transformed_error) = operation('{"who": "anciano"}') assert "who" in transformed_output assert transformed_output["who"] == "anciano" assert transformed_error is None
def test_reader_json_empty(): operation = sequential(from_json, keep(fields=["who"])) (transformed_output, transformed_error) = operation(None) assert transformed_output is None assert transformed_error == {'who': "Can't parse"}
def etl(): return sequential( csv_to_map([ 'date', 'file', 'date2', 'log', 'app', 'beat', 'front', 'is_log', 'msg', 'offset', 'arch' ]), keep(["msg"]))