def test_normal_complex_json(self):
db_path = "test_complex_json.sqlite"
runner = CliRunner()
with runner.isolated_filesystem():
file_path = valid_complex_json_file()
result = runner.invoke(cmd, ["-o", db_path, "file", file_path])
print_traceback(result)
assert result.exit_code == ExitCode.SUCCESS
con = SimpleSQLite(db_path, "r")
expected = {
"ratings",
"screenshots_4",
"screenshots_3",
"screenshots_5",
"screenshots_1",
"screenshots_2",
"tags",
"versions",
"root",
SourceInfo.get_table_name(),
}
assert set(con.fetch_table_names()) == expected
def test_normal_type_hint_header(self):
text = dedent("""\
"a text","b integer","c real"
1,"1","1.1"
2,"2","1.2"
3,"3","1.3"
""")
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(
cmd,
["--type-hint-header", "-o", self.db_path, "stdin", "csv"],
input=text)
print_traceback(result)
assert result.exit_code == ExitCode.SUCCESS
con = SimpleSQLite(self.db_path, "r")
table_names = list(
set(con.fetch_table_names()) - {SourceInfo.get_table_name()})
# table name may change test execution order
tbldata = con.select_as_tabledata(table_names[0])
assert tbldata.headers == ["a text", "b integer", "c real"]
assert tbldata.rows == [("1", 1, 1.1), ("2", 2, 1.2),
("3", 3, 1.3)]
def test_normal_json(self):
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(cmd, ["-o", self.db_path, "stdin", "json"],
input=complex_json)
print_traceback(result)
assert result.exit_code == ExitCode.SUCCESS
con = SimpleSQLite(self.db_path, "r")
expected = {
"ratings",
"screenshots_4",
"screenshots_3",
"screenshots_5",
"screenshots_1",
"screenshots_2",
"tags",
"versions",
"root",
SourceInfo.get_table_name(),
}
assert set(con.fetch_table_names()) == expected
result = runner.invoke(cmd, ["-o", self.db_path, "stdin", "csv"],
input=complex_json)
assert result.exit_code == ExitCode.FAILED_CONVERT
def test_normal_type_hint_header(self):
url = "https://example.com/type_hint_header.csv"
responses.add(
responses.GET,
url,
body=dedent("""\
"a text","b integer","c real"
1,"1","1.1"
2,"2","1.2"
3,"3","1.3"
"""),
content_type="text/plain; charset=utf-8",
status=200,
)
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(
cmd, ["--type-hint-header", "-o", self.db_path, "url", url])
print_traceback(result)
assert result.exit_code == ExitCode.SUCCESS
con = SimpleSQLite(self.db_path, "r")
table_names = list(
set(con.fetch_table_names()) - {SourceInfo.get_table_name()})
# table name may change test execution order
tbldata = con.select_as_tabledata(table_names[0])
assert tbldata.headers == ["a text", "b integer", "c real"]
assert tbldata.rows == [("1", 1, 1.1), ("2", 2, 1.2),
("3", 3, 1.3)]
def test_normal_json(self):
url = "https://example.com/complex_json.json"
responses.add(
responses.GET,
url,
body=complex_json,
content_type="text/plain; charset=utf-8",
status=200,
)
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(cmd, ["-o", self.db_path, "url", url])
print_traceback(result)
assert result.exit_code == ExitCode.SUCCESS
con = SimpleSQLite(self.db_path, "r")
expected = {
"ratings",
"screenshots_4",
"screenshots_3",
"screenshots_5",
"screenshots_1",
"screenshots_2",
"tags",
"versions",
"root",
SourceInfo.get_table_name(),
}
assert set(con.fetch_table_names()) == expected
def test_normal_multi_table(self, con_a0, con_b0):
out_db_path = "test.sqlite"
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(cmd, [
"-o", out_db_path, "file", con_a0.database_path,
con_b0.database_path
])
print_traceback(result)
assert result.exit_code == ExitCode.SUCCESS
expected_list = [
TableData(TEST_TABLE_NAME_A, ["attr_a", "attr_b"],
[[1, 2], [3, 4]]),
TableData(TEST_TABLE_NAME_B, ["ba", "bb"],
[[101, 102], [103, 104]]),
]
for tabledata in SqliteFileLoader(out_db_path).load():
if tabledata.table_name == SourceInfo.get_table_name():
continue
print("[actual]\n{}".format(tabledata))
for record in tabledata.value_matrix:
print(" {}".format(record))
assert tabledata in expected_list
def test_normal_multi_file_same_table_same_structure(self):
db_path = "test.sqlite"
runner = CliRunner()
with runner.isolated_filesystem():
files = [valid_json_multi_file_2_1(), valid_json_multi_file_2_2()]
result = runner.invoke(cmd, ["-o", db_path, "file"] + files)
print_traceback(result)
assert result.exit_code == ExitCode.SUCCESS
con = SimpleSQLite(db_path, "r")
expected_tables = ["multij2", SourceInfo.get_table_name()]
actual_tables = con.fetch_table_names()
print_test_result(expected=expected_tables, actual=actual_tables)
assert set(actual_tables) == set(expected_tables)
expected_data_table = {
"multij2": [
(1, 4.0, "a"),
(2, 2.1, "bb"),
(3, 120.9, "ccc"),
(1, 4.0, "a"),
(2, 2.1, "bb"),
(3, 120.9, "ccc"),
]
}
for table in con.fetch_table_names():
if table == SourceInfo.get_table_name():
continue
expected_data = expected_data_table.get(table)
actual_data = con.select("*", table_name=table).fetchall()
message = "table={}, expected={}, actual={}".format(
table, expected_data, actual_data)
print("--- table: {} ---".format(table))
print_test_result(expected=expected_data, actual=actual_data)
assert expected_data == actual_data, message
def test_normal_same_table(self, con_a0, con_a1):
out_db_path = "test.sqlite"
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(
cmd, ["-o", out_db_path, "file", con_a0.database_path, con_a1.database_path]
)
print_traceback(result)
assert result.exit_code == ExitCode.SUCCESS
expected = TableData(
TEST_TABLE_NAME_A, ["attr_a", "attr_b"], [[1, 2], [3, 4], [11, 12], [13, 14]]
)
for tabledata in SqliteFileLoader(out_db_path).load():
if tabledata.table_name == SourceInfo.get_table_name():
continue
assert tabledata == expected
def test_normal_append(self):
db_path = "test.sqlite"
runner = CliRunner()
with runner.isolated_filesystem():
files = [valid_json_multi_file_2_1()]
table_name = "multij2"
expected_tables = [table_name, SourceInfo.get_table_name()]
# first execution without --append option (new) ---
result = runner.invoke(cmd, ["-o", db_path, "file"] + files)
print_traceback(result)
assert result.exit_code == ExitCode.SUCCESS
con = SimpleSQLite(db_path, "r")
actual_tables = con.fetch_table_names()
print_test_result(expected=expected_tables, actual=actual_tables)
assert set(actual_tables) == set(expected_tables)
actual_data = con.select("*", table_name=table_name).fetchall()
expected_data = [(1, 4.0, "a"), (2, 2.1, "bb"), (3, 120.9, "ccc")]
print_test_result(expected=expected_data, actual=actual_data)
assert expected_data == actual_data
# second execution with --append option ---
result = runner.invoke(cmd,
["-o", db_path, "--append", "file"] + files)
print_traceback(result)
assert result.exit_code == ExitCode.SUCCESS
con = SimpleSQLite(db_path, "r")
actual_tables = con.fetch_table_names()
print_test_result(expected=expected_tables, actual=actual_tables)
assert set(actual_tables) == set(expected_tables)
actual_data = con.select("*", table_name=table_name).fetchall()
expected_data = [
(1, 4.0, "a"),
(2, 2.1, "bb"),
(3, 120.9, "ccc"),
(1, 4.0, "a"),
(2, 2.1, "bb"),
(3, 120.9, "ccc"),
]
print_test_result(expected=expected_data, actual=actual_data)
assert expected_data == actual_data
# third execution without --append option (overwrite) ---
result = runner.invoke(cmd, ["-o", db_path, "file"] + files)
print_traceback(result)
assert result.exit_code == ExitCode.SUCCESS
con = SimpleSQLite(db_path, "r")
actual_tables = con.fetch_table_names()
print_test_result(expected=expected_tables, actual=actual_tables)
assert set(actual_tables) == set(expected_tables)
actual_data = con.select("*", table_name=table_name).fetchall()
expected_data = [(1, 4.0, "a"), (2, 2.1, "bb"), (3, 120.9, "ccc")]
print_test_result(expected=expected_data, actual=actual_data)
assert expected_data == actual_data
def test_normal_multi_file_different_table(self):
db_path = "test.sqlite"
runner = CliRunner()
with runner.isolated_filesystem():
files = [
valid_json_single_file(),
valid_json_single_b_file(),
valid_json_multi_file_1(),
valid_json_kv_file(),
valid_csv_file_1_1(),
valid_csv_file_2_1(),
invalid_csv_file(),
valid_tsv_file(),
invalid_tsv_file(),
valid_excel_file(),
invalid_excel_file_1(),
invalid_excel_file_2(),
valid_html_file(),
invalid_html_file(),
valid_ltsv_file(),
invalid_ltsv_file(),
valid_markdown_file(),
not_supported_format_file(),
]
result = runner.invoke(cmd, ["-o", db_path, "file"] + files)
assert result.exit_code == ExitCode.SUCCESS
con = SimpleSQLite(db_path, "r")
expected_tables = [
"2018Asset",
"singlejson",
"multij1",
"multij2",
"valid_kv",
"csv_a",
"rename_insert",
"excel_sheet_a",
"excel_sheet_c",
"excel_sheet_d",
"valid_ltsv_a",
"testtitle_tablename",
"testtitle_html2",
"tsv_a",
"valid_mdtable_markdown1",
"root",
SourceInfo.get_table_name(),
]
actual_tables = con.fetch_table_names()
print_test_result(expected=expected_tables, actual=actual_tables)
assert set(actual_tables) == set(expected_tables)
expected_data_table = {
"2018Asset": [(1, "1billion", "item_entertainment_18_212")],
"singlejson": [(1, 4.0, "a"), (2, 2.1, "bb"),
(3, 120.9, "ccc")],
"multij1": [(1, 4.0, "a"), (2, 2.1, "bb"), (3, 120.9, "ccc")],
"multij2": [(1, 4.0), (2, None), (3, 120.9)],
"valid_kv": [("json_b", "hoge"), ("json_c", "bar")],
"csv_a": [(1, 4.0, "a"), (2, 2.1, "bb"), (3, 120.9, "ccc")],
"rename_insert": [
(1, 55, "D Sam", 31, "Raven"),
(2, 36, "J Ifdgg", 30, "Raven"),
(3, 91, "K Wedfb", 28, "Raven"),
],
"excel_sheet_a": [(1.0, 1.1, "a"), (2.0, 2.2, "bb"),
(3.0, 3.3, "cc")],
"excel_sheet_c": [(1, 1.1, "a"), (2, "", "bb"), (3, 3.3, "")],
"excel_sheet_d": [(1, 1.1, "a"), (2, "", "bb"), (3, 3.3, "")],
"testtitle_tablename": [(1, 123.1, "a"), (2, 2.2, "bb"),
(3, 3.3, "ccc")],
"valid_ltsv_a": [
(1, 123.1, '"ltsv0"', 1.0, '"1"'),
(2, 2.2, '"ltsv1"', 2.2, '"2.2"'),
(3, 3.3, '"ltsv2"', 3.0, '"cccc"'),
],
"testtitle_html2": [(1, 123.1), (2, 2.2), (3, 3.3)],
"tsv_a": [(1, 4.0, "tsv0"), (2, 2.1, "tsv1"),
(3, 120.9, "tsv2")],
"valid_mdtable_markdown1": [(1, 123.1, "a"), (2, 2.2, "bb"),
(3, 3.3, "ccc")],
}
for table in con.fetch_table_names():
if table in (SourceInfo.get_table_name(), "root"):
continue
result = con.select("*", table_name=table)
expected_data = expected_data_table.get(table)
actual_data = result.fetchall()
message = "table={}, expected={}, actual={}".format(
table, expected_data, actual_data)
print("--- table: {} ---".format(table))
print_test_result(expected=expected_data, actual=actual_data)
assert sorted(expected_data) == sorted(actual_data), message