def test_stop_on_errors():
""" Unit test cusrom ParseFixer.stop_on_errors
"""
# fmt: off
table_lines = [
["**tab_ok"],
["dst1"],
[ "a1" , "a2" , "a3" , "a4" ],
[ "-" , "-" , "-" , "-" ],
[ 1 , 2 , 3 , 3.14 ],
[],
["**tab_errors"],
["dst1"],
[ "a1" , "a2" , "a3" , "a4" ],
[ "-" , "-" , "-" , "-" ],
[ "NaN" , "nan" , "Nine", "Ten" ],
[ 1 , 2 , 3 , 4 ],
]
# fmt: on
fix = ParseFixer()
fix.stop_on_errors = True
fix._dbg = False # ignore during test
pi = 0
with pytest.raises(ValueError):
for typ, tab in parse_blocks(table_lines, fixer=fix, to="pdtable"):
if typ != BlockType.TABLE:
continue
assert tab.df["a4"][0] == 3.14
pi += 1
with pytest.raises(ValueError):
for typ, tab in parse_blocks(table_lines, fixer=fix, to="jsondata"):
if typ != BlockType.TABLE:
continue
assert tab["columns"]["a4"]["values"][0] == 3.14
pi += 1
# cellgrid does not parse values, i.e. no ValueError
for typ, tab in parse_blocks(table_lines, fixer=fix, to="cellgrid"):
if typ != BlockType.TABLE:
continue
if tab[0][0] == "**tab_ok":
assert tab[4][3] == 3.14
pi += 1
if tab[0][0] == "**tab_errors":
assert tab[4][3] == "Ten"
assert pi == 3 # 😉
def test_json_pdtable():
""" ensure dict-obj to pdtable conversion
compare to target created w. read_stream_csv
"""
cell_rows = [
line.split(";") for line in dedent("""\
**farm_types1;;;
your_farm my_farm farms_galore;;;
species; num; flt; log;
text; -; kg; onoff;
chicken; 2; 3; 1;
pig; 4; 39; 0;
goat; 4; -; 1;
zybra; 4; -; 0;
cow; NaN; 200; 1;
goose; 2; 9; 0;
""").strip().split("\n")
]
pandas_pdtab = None
# with io.StringIO(csv_src) as fh:
g = parse_blocks(cell_rows,
**{"origin": '"types1.csv" row 1'},
fixer=custom_test_fixer)
for tp, tab in g:
pandas_pdtab = tab
# fmt: off
table_json_data = {
"name": "farm_types1",
"columns": {
"species": {
"unit": "text",
"values": ["chicken", "pig", "goat", "zybra", "cow", "goose"]
},
"num": {
"unit": "-",
"values": [2.0, 4.0, 4.0, 4.0, None, 2.0]
},
"flt": {
"unit": "kg",
"values": [3.0, 39.0, None, None, 200.0, 9.0]
},
"log": {
"unit": "onoff",
"values": [True, False, True, False, True, False]
}
},
"destinations": {
"your_farm": None,
"my_farm": None,
"farms_galore": None
}
}
# fmt: on
json_pdtab = json_data_to_table(table_json_data, fixer=custom_test_fixer)
assert pandas_pdtab.equals(json_pdtab)
def test_fat():
""" Factory Acceptance Test
Read input files as dictionary objects.
Compare objects to stored target objects (input_dir() / all.json)
"""
all_files = 0
for fn in os.listdir(input_dir()):
path = input_dir() / fn
if not os.path.isfile(path):
continue
if fn in ["auto_fixed.py", "__init__.py", "all.json", "all.csv"]:
continue
all_files += 1
with open(input_dir() / "all.json") as f:
all_json = json.load(f)
count = 0
for fn in os.listdir(input_dir()):
path = input_dir() / fn
if not os.path.isfile(path):
continue
if fn in ["auto_fixed.py", "__init__.py", "all.json", "all.csv"]:
continue
with open(input_dir() / fn, "r") as fh:
cell_rows = (line.rstrip("\n").split(";") for line in fh)
g = parse_blocks(cell_rows,
**{
"origin": f'"{fn}"',
"to": "jsondata"
},
fixer=custom_test_fixer)
for tp, tt in g:
if tp == BlockType.TABLE:
""" compare generic object
i.e. containing None instead of pd.NaT, np.nan &c.
"""
count += 1
assert tt == all_json[fn]
assert count == all_files
def test_stop_on_errors_default_fixer():
""" Unit test ParseFixer: raise ValueError on empty float and empty onoff
"""
# fmt: off
table_lines_flt = [
["**tab_ok"],
["dst1"],
[ "a1" , "a2" , "a3" , "a4" ],
[ "-" , "-" , "-" , "-" ],
[ 1 , 2 , 3 , 3.14 ],
[],
["**tab_errors"],
["dst1"],
[ "a1" , "a2" , "a3" , "a4" ],
[ "-" , "-" , "-" , "-" ],
[ 1 , 2 , 3 , "" ],
]
# fmt: on
g = parse_blocks(table_lines_flt, filter=lambda ty, tn: ty == BlockType.TABLE)
typ, tab = next(g)
assert tab.name == "tab_ok"
assert tab.df["a4"][0] == 3.14
with pytest.raises(ValueError):
typ, tab = next(g)
print(f"-oOo-: {typ} {tab}")
# fmt: off
table_lines_onoff = [
["**tab_ok"],
["dst1"],
[ "a1" , "a2"],
[ "-" , "onoff"],
[ 3.14 , 1 ],
[],
["**tab_errors"],
["dst1"],
[ "a1" , "a2" ],
[ "-" , "onoff" ],
[ 1 , None ],
]
# fmt: on
g = parse_blocks(table_lines_onoff, filter=lambda ty, tn: ty == BlockType.TABLE)
typ, tab = next(g)
assert tab.name == "tab_ok"
assert tab.df["a1"][0] == 3.14
with pytest.raises(ValueError):
typ, tab = next(g)
print(f"-oOo-: {typ} {tab}")
# fmt: off
table_lines_datetime = [
["**tab_ok"],
["dst1"],
[ "a1" , "a2"],
[ "-" , "datetime" ],
[ 14 , pd.to_datetime("2020-08-11")],
[ 3.14 , "2020-08-12" ],
[],
["**tab_errors"],
["dst1"],
[ "a1" , "a2" ],
[ "-" , "datetime" ],
[ 14 , pd.to_datetime("2020-08-13")],
[ 1 , None ],
]
# fmt: on
g = parse_blocks(table_lines_datetime, filter=lambda ty, tn: ty == BlockType.TABLE)
typ, tab = next(g)
assert tab.name == "tab_ok"
assert tab.df["a1"][1] == 3.14
assert tab.df["a2"][1] == pd.to_datetime("2020-08-12")
with pytest.raises(ValueError):
typ, tab = next(g)
print(f"-oOo-: {typ} {tab}")