def test_base_fragment_reused_chunking(local_engine_empty):
# Check that if we split a table into chunks and some chunks are the same, they get assigned to the same objects.
OUTPUT.init()
base = OUTPUT.head
_make_test_table(OUTPUT)
OUTPUT.commit(chunk_size=5)
table_1 = OUTPUT.head.get_table("test")
# Table 1 produced 3 objects
assert len(OUTPUT.objects.get_all_objects()) == 3
# All chunks are the same
base.checkout()
_make_test_table(OUTPUT)
OUTPUT.commit(chunk_size=5)
table_2 = OUTPUT.head.get_table("test")
assert len(OUTPUT.objects.get_all_objects()) == 3
assert table_1.objects == table_2.objects
# Insert something else into the middle chunk so that it's different. This will get conflated so won't get recorded
# as an update.
base.checkout()
_make_test_table(OUTPUT)
OUTPUT.run_sql(
"UPDATE test SET value_1 = 'UPDATED', value_2 = 42 WHERE key = 7")
OUTPUT.commit(chunk_size=5)
table_3 = OUTPUT.head.get_table("test")
assert len(OUTPUT.objects.get_all_objects()) == 4
# Table 3 reused the first and the last object but created a new one for the middle fragment.
assert len(table_3.objects) == 3
assert table_3.objects[0] == table_1.objects[0]
assert table_3.objects[1] != table_1.objects[1]
assert table_3.objects[2] == table_1.objects[2]
def test_import_bare(pg_repo_local):
# Check import without checking anything out, just by manipulating metadata and running LQs against
# source images.
# Create a new schema and import 'fruits'
OUTPUT.init()
# Make sure the existing table is preserved.
OUTPUT.run_sql("CREATE TABLE sentinel (key INTEGER)")
OUTPUT.commit()
pg_repo_local.uncheckout()
OUTPUT.uncheckout()
OUTPUT.import_tables(
tables=["imported_fruits"],
source_repository=pg_repo_local,
image_hash=pg_repo_local.images["latest"].image_hash,
source_tables=["SELECT * FROM fruits WHERE fruit_id = 1"],
parent_hash=OUTPUT.images["latest"].image_hash,
do_checkout=False,
table_queries=[True],
)
assert OUTPUT.head is None
assert pg_repo_local.head is None
assert sorted(OUTPUT.images["latest"].get_tables()) == [
"imported_fruits", "sentinel"
]
assert list(OUTPUT.images["latest"].get_table("imported_fruits").query(
columns=["name"], quals=[])) == [{
"name": "apple"
}]
def test_splitfile_end_to_end_with_uploading(local_engine_empty, remote_engine,
pg_repo_remote_multitag,
mg_repo_remote, clean_minio):
# An end-to-end test:
# * Create a derived dataset from some tables imported from the remote engine
# * Push it back to the remote engine, uploading all objects to S3 (instead of the remote engine itself)
# * Delete everything from pgcache
# * Run another splitfile that depends on the just-pushed dataset (and does lazy checkouts to
# get the required tables).
# Do the same setting up first and run the splitfile against the remote data.
execute_commands(load_splitfile("import_remote_multiple.splitfile"),
params={"TAG": "v1"},
output=OUTPUT)
remote_output = Repository(OUTPUT.namespace, OUTPUT.repository,
remote_engine)
# Push with upload
OUTPUT.push(remote_repository=remote_output,
handler="S3",
handler_options={})
# Unmount everything locally and cleanup
for mountpoint, _ in get_current_repositories(local_engine_empty):
mountpoint.delete()
OUTPUT.objects.cleanup()
stage_2 = R("output_stage_2")
execute_commands(
load_splitfile("import_from_preuploaded_remote.splitfile"),
output=stage_2)
assert stage_2.run_sql("SELECT id, name, fruit, vegetable FROM diet") == [
(2, "James", "orange", "carrot")
]
def test_splitfile_rebuild_update(local_engine_empty, pg_repo_remote_multitag):
runner = CliRunner()
result = runner.invoke(
build_c,
[
RESOURCES + "import_remote_multiple.splitfile", "-a", "TAG", "v1",
"-o", "output"
],
)
assert result.exit_code == 0
# Rerun the output:latest against v2 of the test/pg_mount
result = runner.invoke(rebuild_c,
["output:latest", "--against", "test/pg_mount:v2"])
output_v2 = OUTPUT.head
assert result.exit_code == 0
v2 = pg_repo_remote_multitag.images["v2"]
assert output_v2.provenance() == [(pg_repo_remote_multitag, v2.image_hash)]
# Now rerun the output:latest against the latest version of everything.
# In this case, this should all resolve to the same version of test/pg_mount (v2) and not produce
# any extra commits.
curr_commits = OUTPUT.images()
result = runner.invoke(rebuild_c, ["output:latest", "-u"])
assert result.exit_code == 0
assert output_v2 == OUTPUT.head
assert OUTPUT.images() == curr_commits
def test_commandline_show_empty_image(local_engine_empty):
# Check size calculations etc in an empty image don't cause errors.
runner = CliRunner()
OUTPUT.init()
assert OUTPUT.images["latest"].get_size() == 0
result = runner.invoke(show_c, [str(OUTPUT) + ":" + "000000000000"],
catch_exceptions=False)
assert "Size: 0.00 B" in result.output
def _setup_dataset():
OUTPUT.init()
OUTPUT.run_sql("""CREATE TABLE test (id integer, name varchar);
INSERT INTO test VALUES (1, 'test')""")
OUTPUT.commit()
OUTPUT.run_sql("INSERT INTO test VALUES (2, 'test2')")
return OUTPUT.commit()
def test_from_remote(local_engine_empty, pg_repo_remote_multitag):
# Test running commands that base new datasets on a remote repository.
execute_commands(load_splitfile("from_remote.splitfile"),
params={"TAG": "v1"},
output=OUTPUT)
new_head = OUTPUT.head
parent = OUTPUT.images.by_hash(new_head.parent_id)
# Go back to the parent: the two source tables should exist there
parent.checkout()
assert OUTPUT.engine.table_exists(OUTPUT.to_schema(), "fruits")
assert OUTPUT.engine.table_exists(OUTPUT.to_schema(), "vegetables")
assert not OUTPUT.engine.table_exists(OUTPUT.to_schema(), "join_table")
new_head.checkout()
assert OUTPUT.engine.table_exists(OUTPUT.to_schema(), "fruits")
assert OUTPUT.engine.table_exists(OUTPUT.to_schema(), "vegetables")
assert OUTPUT.run_sql("SELECT * FROM join_table") == [
(1, "apple", "potato"),
(2, "orange", "carrot"),
]
# Now run the same splitfile but from the v2 of the remote (where row 1 has been removed from the fruits table)
# First, remove the output mountpoint (the executor tries to fetch the commit 0000 from it otherwise which
# doesn't exist).
OUTPUT.delete()
execute_commands(load_splitfile("from_remote.splitfile"),
params={"TAG": "v2"},
output=OUTPUT)
assert OUTPUT.run_sql("SELECT * FROM join_table") == [(2, "orange",
"carrot")]
def test_local_import_splitfile(pg_repo_local):
execute_commands(load_splitfile("import_local.splitfile"), output=OUTPUT)
head = OUTPUT.head
old_head = head.parent_id
OUTPUT.images.by_hash(old_head).checkout()
assert not OUTPUT.engine.table_exists(OUTPUT.to_schema(), "my_fruits")
assert not OUTPUT.engine.table_exists(OUTPUT.to_schema(), "fruits")
head.checkout()
assert OUTPUT.engine.table_exists(OUTPUT.to_schema(), "my_fruits")
assert not OUTPUT.engine.table_exists(OUTPUT.to_schema(), "fruits")
def test_update_without_import_splitfile(pg_repo_local):
# Test that correct commits are produced by executing an splitfile (both against newly created and already
# existing tables on an existing mountpoint)
execute_commands(load_splitfile("update_without_import.splitfile"),
output=OUTPUT)
log = OUTPUT.head.get_log()
log[1].checkout()
assert OUTPUT.run_sql("SELECT * FROM my_fruits") == []
log[0].checkout()
assert OUTPUT.run_sql("SELECT * FROM my_fruits") == [(1, "pineapple")]
def test_splitfile_sql_failure(local_engine_empty, pg_repo_remote_multitag):
assert len(OUTPUT.images()) == 0
assert _get_table_count(OUTPUT) == 0
with pytest.raises(psycopg2.errors.UndefinedTable) as e:
execute_commands(load_splitfile("import_remote_broken_stage_2.splitfile"), output=OUTPUT)
assert 'relation "nonexistent_fruits_table" does not exist' in str(e.value)
# Check the execution created the first dummy (000...) image and the second image
# with IMPORT results
assert len(OUTPUT.images()) == 2
assert _get_table_count(OUTPUT) == 2
assert sorted(OUTPUT.images["latest"].get_tables()) == ["my_fruits", "vegetables"]
def test_from_remote_hash(local_engine_empty, pg_repo_remote):
head = pg_repo_remote.head.image_hash
# Test running commands that base new datasets on a remote repository.
execute_commands(load_splitfile("from_remote.splitfile"),
params={"TAG": head[:10]},
output=OUTPUT)
assert OUTPUT.engine.table_exists(OUTPUT.to_schema(), "fruits")
assert OUTPUT.engine.table_exists(OUTPUT.to_schema(), "vegetables")
assert OUTPUT.run_sql("SELECT * FROM join_table") == [
(1, "apple", "potato"),
(2, "orange", "carrot"),
]
def test_basic_splitfile(pg_repo_local):
execute_commands(load_splitfile("create_table.splitfile"), output=OUTPUT)
log = list(reversed(OUTPUT.head.get_log()))
log[1].checkout()
assert OUTPUT.run_sql("SELECT * FROM my_fruits") == []
log[2].checkout()
assert OUTPUT.run_sql("SELECT * FROM my_fruits") == [(1, "pineapple")]
log[3].checkout()
assert OUTPUT.run_sql("SELECT * FROM my_fruits") == [(1, "pineapple"),
(2, "banana")]
def test_splitfile_cached(pg_repo_local):
# Check that no new commits/snaps are created if we rerun the same splitfile
execute_commands(
load_splitfile("import_local_multiple_with_queries.splitfile"),
output=OUTPUT)
images = OUTPUT.images()
assert len(images) == 4
execute_commands(
load_splitfile("import_local_multiple_with_queries.splitfile"),
output=OUTPUT)
new_images = OUTPUT.images()
assert new_images == images
def test_import_and_update(local_engine_empty, unprivileged_pg_repo):
OUTPUT.init()
head = OUTPUT.head
remote_head = unprivileged_pg_repo.images["latest"]
# Import the 'fruits' table from the origin.
import_table_from_remote(unprivileged_pg_repo, ["fruits"],
remote_head.image_hash,
OUTPUT,
target_tables=[])
new_head = OUTPUT.head
OUTPUT.run_sql("INSERT INTO fruits VALUES (3, 'mayonnaise')")
new_head_2 = OUTPUT.commit()
head.checkout()
assert not OUTPUT.engine.table_exists(OUTPUT.to_schema(), "fruits")
new_head.checkout()
assert OUTPUT.run_sql("SELECT * FROM fruits") == [(1, "apple"),
(2, "orange")]
new_head_2.checkout()
assert OUTPUT.run_sql("SELECT * FROM fruits") == [
(1, "apple"),
(2, "orange"),
(3, "mayonnaise"),
]
def test_import_basic(pg_repo_local):
# Create a new schema and import 'fruits' from the mounted PG table.
OUTPUT.init()
head = OUTPUT.head
OUTPUT.import_tables(tables=["imported_fruits"],
source_repository=pg_repo_local,
source_tables=["fruits"])
assert OUTPUT.run_sql("SELECT * FROM imported_fruits"
) == pg_repo_local.run_sql("SELECT * FROM fruits")
new_head = OUTPUT.head
assert new_head != head
assert new_head.parent_id == head.image_hash
def test_import_multiple_tables(pg_repo_local):
OUTPUT.init()
head = OUTPUT.head
OUTPUT.import_tables(tables=[],
source_repository=pg_repo_local,
source_tables=[])
for table_name in ["fruits", "vegetables"]:
assert OUTPUT.run_sql("SELECT * FROM %s" %
table_name) == pg_repo_local.run_sql(
"SELECT * FROM %s" % table_name)
new_head = OUTPUT.head
assert new_head != head
assert new_head.parent_id == head.image_hash
def test_import_all(local_engine_empty):
execute_commands(load_splitfile("import_all_from_mounted.splitfile"),
output=OUTPUT)
head = OUTPUT.head
old_head = OUTPUT.images.by_hash(head.parent_id)
old_head.checkout()
tables = ["vegetables", "fruits"]
contents = [[(1, "potato"), (2, "carrot")], [(1, "apple"), (2, "orange")]]
for t in tables:
assert not OUTPUT.engine.table_exists(OUTPUT.to_schema(), t)
head.checkout()
for t, c in zip(tables, contents):
assert OUTPUT.run_sql("SELECT * FROM %s" % t) == c
def test_splitfile_object_download_failure(local_engine_empty, pg_repo_remote_multitag):
# Simulate an object download failure (that happens inside of the engine during IMPORT
# execution) propagating to the caller and not leaving the engine in an inconsistent state.
object_id = pg_repo_remote_multitag.images["v1"].get_table("fruits").objects[0]
assert object_id == "o0e742bd2ea4927f5193a2c68f8d4c51ea018b1ef3e3005a50727147d2cf57b"
tmp_object_id = "o" + "0" * 62
pg_repo_remote_multitag.engine.run_sql(
SQL("ALTER TABLE splitgraph_meta.{} RENAME TO {}").format(
Identifier(object_id), Identifier(tmp_object_id)
)
)
assert len(OUTPUT.images()) == 0
assert _get_table_count(OUTPUT) == 0
with pytest.raises(ObjectCacheError) as e:
execute_commands(
load_splitfile("import_remote_multiple.splitfile"), params={"TAG": "v1"}, output=OUTPUT
)
assert "Missing 1 object (%s)" % object_id in str(e.value)
# Check the execution didn't create the image
assert len(OUTPUT.images()) == 0
assert _get_table_count(OUTPUT) == 0
# Rename the object back and retry the Splitfile
pg_repo_remote_multitag.engine.run_sql(
SQL("ALTER TABLE splitgraph_meta.{} RENAME TO {}").format(
Identifier(tmp_object_id), Identifier(object_id)
)
)
execute_commands(
load_splitfile("import_remote_multiple.splitfile"), params={"TAG": "v1"}, output=OUTPUT
)
OUTPUT.head.checkout()
assert OUTPUT.run_sql("SELECT id, fruit, vegetable FROM join_table") == [
(1, "apple", "potato"),
(2, "orange", "carrot"),
]
assert len(OUTPUT.images()) == 3
# 2 tables in the first non-empty image, 3 tables in the second image
# (previous 2 + joined data).
assert _get_table_count(OUTPUT) == 5
def test_splitfile_remote(local_engine_empty, pg_repo_remote_multitag):
# We use the v1 tag when importing from the remote, so fruit_id = 1 still exists there.
execute_commands(load_splitfile("import_remote_multiple.splitfile"),
params={"TAG": "v1"},
output=OUTPUT)
assert OUTPUT.run_sql("SELECT id, fruit, vegetable FROM join_table") == [
(1, "apple", "potato"),
(2, "orange", "carrot"),
]
# Now run the commands against v2 and make sure the fruit_id = 1 has disappeared from the output.
execute_commands(load_splitfile("import_remote_multiple.splitfile"),
params={"TAG": "v2"},
output=OUTPUT)
assert OUTPUT.run_sql("SELECT id, fruit, vegetable FROM join_table") == [
(2, "orange", "carrot")
]
def test_splitfile_inline_sql(readonly_pg_repo, pg_repo_local):
# Test SQL commands accessing repos directly -- join a remote repo with
# some local data.
prepare_lq_repo(pg_repo_local, commit_after_every=False, include_pk=True)
pg_repo_local.head.tag("v2")
execute_commands(
load_splitfile("inline_sql.splitfile"),
output=OUTPUT,
)
new_head = OUTPUT.head
new_head.checkout()
assert new_head.get_tables() == ["balanced_diet"]
assert OUTPUT.run_sql("SELECT * FROM balanced_diet") == [
(1, "apple", None, "potato"),
(2, "orange", datetime.datetime(2019, 1, 1, 12, 0), "carrot"),
]
local_repo_head = pg_repo_local.head.image_hash
other_repo_head = readonly_pg_repo.images["latest"].image_hash
assert new_head.provenance_data == [
{
"sources": [
{
"source": "pg_mount",
"source_hash": other_repo_head,
"source_namespace": "otheruser",
},
{
"source": "pg_mount",
"source_hash": local_repo_head,
"source_namespace": "test"
},
],
"sql": ("CREATE TABLE balanced_diet\n"
" AS SELECT fruits.fruit_id AS id\n"
" , fruits.name AS fruit\n"
" , my_fruits.timestamp AS timestamp\n"
" , vegetables.name AS vegetable\n"
" FROM "
'"otheruser/pg_mount:{0}".fruits '
"AS fruits\n"
" INNER JOIN "
'"otheruser/pg_mount:{0}".vegetables '
"AS vegetables ON fruits.fruit_id = vegetable_id\n"
" LEFT JOIN "
'"test/pg_mount:{1}".fruits '
"AS my_fruits ON my_fruits.fruit_id = fruits.fruit_id;\n"
"\n"
"ALTER TABLE balanced_diet ADD PRIMARY KEY (id)").format(
other_repo_head, local_repo_head),
"type":
"SQL",
},
]
def test_import_from_remote(local_engine_empty, unprivileged_pg_repo):
# Start with a clean repo -- add a table to output to see if it's preserved.
head = _setup_dataset()
local_objects = OUTPUT.objects
assert len(local_objects.get_downloaded_objects()) == 2
assert len(local_objects.get_all_objects()) == 2
assert local_engine_empty.get_all_tables(OUTPUT.to_schema()) == ["test"]
# Import the 'fruits' table from the origin.
remote_head = unprivileged_pg_repo.images["latest"]
import_table_from_remote(unprivileged_pg_repo, ["fruits"],
remote_head.image_hash,
OUTPUT,
target_tables=[])
new_head = OUTPUT.head
# Check that the table now exists in the output, is committed and there's no trace of the cloned repo.
# Also clean up the unused objects to make sure that the newly cloned table is still recorded.
assert sorted(local_engine_empty.get_all_tables(
OUTPUT.to_schema())) == ["fruits", "test"]
local_objects.cleanup()
assert len(get_current_repositories(local_engine_empty)) == 1
head.checkout()
assert local_engine_empty.table_exists(OUTPUT.to_schema(), "test")
assert not local_engine_empty.table_exists(OUTPUT.to_schema(), "fruits")
new_head.checkout()
assert local_engine_empty.table_exists(OUTPUT.to_schema(), "test")
assert local_engine_empty.table_exists(OUTPUT.to_schema(), "fruits")
assert OUTPUT.run_sql("SELECT * FROM fruits") == [(1, "apple"),
(2, "orange")]
def test_bloom_index_post_factum(local_engine_empty):
OUTPUT.init()
OUTPUT.run_sql(
"CREATE TABLE test (key INTEGER PRIMARY KEY, value_1 TIMESTAMP)")
for i in range(50):
OUTPUT.run_sql("INSERT INTO test VALUES (%s, %s)",
(i + 1, dt(2015, 1, 1) + timedelta(days=i)))
head = OUTPUT.commit()
# Reindex the table after committing it and check that object metadata gets overwritten.
head.get_table("test").reindex(
extra_indexes={"bloom": {
"value_1": {
"probability": 0.01
}
}})
obj = head.get_table("test").objects[0]
object_index = OUTPUT.objects.get_object_meta([obj])[obj].object_index
assert "bloom" in object_index
# Run a snippet from the previous test to check the bloom metadata is valid.
objects = head.get_table("test").objects
for i in range(0, 50, 5):
assert (filter_bloom_index(OUTPUT.engine, objects, [[
("value_1", "=", dt(2015, 1, 1) + timedelta(days=i))
]]) == objects)
def test_bloom_index_datetime(local_engine_empty):
OUTPUT.init()
OUTPUT.run_sql(
"CREATE TABLE test (key INTEGER PRIMARY KEY, value_1 TIMESTAMP)")
for i in range(50):
OUTPUT.run_sql("INSERT INTO test VALUES (%s, %s)",
(i + 1, dt(2015, 1, 1) + timedelta(days=i)))
head = OUTPUT.commit(
extra_indexes={"test": {
"bloom": {
"value_1": {
"probability": 0.01
}
}
}})
objects = head.get_table("test").objects
# Datetimes are supported in the bloom index if they're passed in as actual datetime
# objects (which Multicorn does).
# Spot check some dates that exist...
for i in range(0, 50, 5):
assert (filter_bloom_index(OUTPUT.engine, objects, [[
("value_1", "=", dt(2015, 1, 1) + timedelta(days=i))
]]) == objects)
# ...and some that don't.
assert (filter_bloom_index(
OUTPUT.engine, objects,
[[("value_1", "=", dt(2015, 1, 1) + timedelta(days=55))]]) == [])
# They also work if passed in as ISO strings with space as a separator.
assert (filter_bloom_index(
OUTPUT.engine, objects,
[[("value_1", "=", "2015-01-01 00:00:00")]]) == objects)
def test_splitfile_remote_hash(local_engine_empty, pg_repo_remote):
head = pg_repo_remote.head.image_hash
execute_commands(load_splitfile("import_remote_multiple.splitfile"),
params={"TAG": head[:10]},
output=OUTPUT)
assert OUTPUT.run_sql(
"SELECT id, fruit, vegetable FROM output.join_table") == [
(1, "apple", "potato"),
(2, "orange", "carrot"),
]
def test_splitfile(local_engine_empty, pg_repo_remote):
runner = CliRunner()
result = runner.invoke(
build_c,
[
RESOURCES + "import_remote_multiple.splitfile",
"-a",
"TAG",
"latest",
"-o",
"output",
],
)
assert result.exit_code == 0
assert OUTPUT.run_sql("SELECT id, fruit, vegetable FROM join_table") == [
(1, "apple", "potato"),
(2, "orange", "carrot"),
]
# Test the sgr provenance command. First, just list the dependencies of the new image.
result = runner.invoke(provenance_c, ["output:latest"])
assert "test/pg_mount:%s" % pg_repo_remote.images[
"latest"].image_hash in result.output
# Second, output the full splitfile (-f)
result = runner.invoke(provenance_c, ["output:latest", "-f"])
assert ("FROM test/pg_mount:%s IMPORT" %
pg_repo_remote.images["latest"].image_hash in result.output)
assert "SQL {CREATE TABLE join_table" in result.output
# Test reverse dependencies
# We're looking at test/pg_mount on the local engine which doesn't exist -- this should fail.
result = runner.invoke(
dependents_c,
[
"test/pg_mount:%s" % pg_repo_remote.images["latest"].image_hash,
],
)
assert result.exit_code == 1
# Now look at test/pg_mount on the remote and look for dependents on the local engine.
result = runner.invoke(
dependents_c,
[
"test/pg_mount:%s" % pg_repo_remote.images["latest"].image_hash,
"--dependents-on",
"LOCAL",
"--source-on",
pg_repo_remote.engine.name,
],
)
assert result.exit_code == 0
assert "is depended on by" in result.output
assert "%s:%s" % (OUTPUT, OUTPUT.head.image_hash) in result.output
def test_splitfile_schema_changes(pg_repo_local, mg_repo_local):
execute_commands(load_splitfile("schema_changes.splitfile"), output=OUTPUT)
old_output_head = OUTPUT.head
# Then, alter the dataset and rerun the splitfile.
pg_repo_local.run_sql("INSERT INTO fruits VALUES (12, 'mayonnaise')")
pg_repo_local.commit()
execute_commands(load_splitfile("schema_changes.splitfile"), output=OUTPUT)
new_output_head = OUTPUT.head
old_output_head.checkout()
assert OUTPUT.run_sql("SELECT * FROM spirit_fruits") == [("James",
"orange", 12)]
new_output_head.checkout()
# Mayonnaise joined with Alex, ID 12 + 10 = 22.
assert OUTPUT.run_sql("SELECT * FROM spirit_fruits") == [
("James", "orange", 12),
("Alex", "mayonnaise", 22),
]
def test_splitfile_with_external_sql(readonly_pg_repo):
# Tests are running from root so we pass in the path to the SQL manually to the splitfile.
execute_commands(
load_splitfile("external_sql.splitfile"),
params={"EXTERNAL_SQL_FILE": RESOURCES + "external_sql.sql"},
output=OUTPUT,
)
assert OUTPUT.run_sql("SELECT id, fruit, vegetable FROM join_table") == [
(1, "apple", "potato"),
(2, "orange", "carrot"),
]
def test_import_query_reuses_hash(pg_repo_local):
OUTPUT.init()
base = OUTPUT.head
# Run two imports: one importing all rows from `fruits` (will reuse the original `fruits` object),
# one importing just the first row (new hash, won't be reused).
ih_v1 = OUTPUT.import_tables(
source_repository=pg_repo_local,
source_tables=[
"SELECT * FROM fruits", "SELECT * FROM fruits WHERE fruit_id = 1"
],
tables=["fruits_all", "fruits_one"],
do_checkout=False,
table_queries=[True, True],
)
v1 = OUTPUT.images.by_hash(ih_v1)
assert v1.get_table("fruits_all").objects == pg_repo_local.head.get_table(
"fruits").objects
assert (len(OUTPUT.objects.get_all_objects()) == 3
) # Original fruits and vegetables + the 1-row import
# Run the same set of imports again: this time both query results already exist and will be reused.
base.checkout()
ih_v2 = OUTPUT.import_tables(
source_repository=pg_repo_local,
source_tables=[
"SELECT * FROM fruits", "SELECT * FROM fruits WHERE fruit_id = 1"
],
tables=["fruits_all", "fruits_one"],
do_checkout=False,
table_queries=[True, True],
)
v2 = OUTPUT.images.by_hash(ih_v2)
assert v2.get_table("fruits_all").objects == v1.get_table(
"fruits_all").objects
assert v2.get_table("fruits_one").objects == v1.get_table(
"fruits_one").objects
assert len(OUTPUT.objects.get_all_objects()
) == 3 # No new objects have been created.
def test_import_with_custom_query(pg_repo_local):
# Test that importing with a custom query creates a new object
pg_repo_local.run_sql("INSERT INTO fruits VALUES (3, 'mayonnaise');"
"INSERT INTO vegetables VALUES (3, 'oregano')")
pg_repo_local.commit()
all_current_objects = pg_repo_local.objects.get_all_objects()
execute_commands(load_splitfile("import_with_custom_query.splitfile"),
output=OUTPUT)
head = OUTPUT.head
old_head = OUTPUT.images.by_hash(head.parent_id)
# First two tables imported as new objects since they had a custom query, the other two get pointed
# to the old pg_repo_local objects.
tables = ["my_fruits", "o_vegetables", "vegetables", "all_fruits"]
contents = [
[(2, "orange")],
[(1, "potato"), (3, "oregano")],
[(1, "potato"), (2, "carrot"), (3, "oregano")],
[(1, "apple"), (2, "orange"), (3, "mayonnaise")],
]
old_head.checkout()
engine = OUTPUT.engine
for t in tables:
assert not engine.table_exists(OUTPUT.to_schema(), t)
head.checkout()
for t, c in zip(tables, contents):
assert sorted(OUTPUT.run_sql("SELECT * FROM %s" % t)) == sorted(c)
for t in tables:
objects = head.get_table(t).objects
if t in ["my_fruits", "o_vegetables"]:
assert all(o not in all_current_objects for o in objects)
else:
assert all(o in all_current_objects for o in objects)
def test_advanced_splitfile(pg_repo_local):
execute_commands(
load_splitfile("import_local_multiple_with_queries.splitfile"),
output=OUTPUT)
assert OUTPUT.engine.table_exists(OUTPUT.to_schema(), "my_fruits")
assert OUTPUT.engine.table_exists(OUTPUT.to_schema(), "vegetables")
assert not OUTPUT.engine.table_exists(OUTPUT.to_schema(), "fruits")
assert OUTPUT.engine.table_exists(OUTPUT.to_schema(), "join_table")
head = OUTPUT.head
old_head = head.parent_id
OUTPUT.images.by_hash(old_head).checkout()
assert not OUTPUT.engine.table_exists(OUTPUT.to_schema(), "join_table")
head.checkout()
assert OUTPUT.run_sql("SELECT id, fruit, vegetable FROM join_table") == [
(2, "orange", "carrot")
]
assert OUTPUT.run_sql("SELECT * FROM my_fruits") == [(2, "orange")]
