def test_gpkg_engine(data_working_copy):
with data_working_copy("points") as (repo_path, wc_path):
engine = Db_GPKG.create_engine(wc_path)
with engine.connect() as db:
r = db.execute(f"SELECT * FROM {H.POINTS.LAYER} LIMIT 1;")
assert r.fetchone() is not None
def test_import_replace_existing(
data_archive,
tmp_path,
cli_runner,
chdir,
):
with data_archive("gpkg-polygons") as data:
repo_path = tmp_path / "emptydir"
r = cli_runner.invoke(["init", repo_path])
assert r.exit_code == 0
with chdir(repo_path):
r = cli_runner.invoke([
"import",
data / "nz-waca-adjustments.gpkg",
"nz_waca_adjustments:mytable",
])
assert r.exit_code == 0, r.stderr
# Now modify the source GPKG
with Db_GPKG.create_engine(
data / "nz-waca-adjustments.gpkg").connect() as conn:
conn.execute(
"UPDATE nz_waca_adjustments SET survey_reference = 'edited' WHERE id = 1424927"
)
r = cli_runner.invoke([
"import",
"--replace-existing",
data / "nz-waca-adjustments.gpkg",
"nz_waca_adjustments:mytable",
])
assert r.exit_code == 0, r.stderr
r = cli_runner.invoke(["show", "-o", "json"])
assert r.exit_code == 0, r.stderr
output = json.loads(r.stdout)
assert output["kart.diff/v1+hexwkb"] == {
"mytable": {
"feature": [{
"-": {
"id": 1424927,
"geom":
"01060000000100000001030000000100000012000000D2B47A3DAEEB65402E86A80212EF42C01D23796880EB6540D54A46E909EE42C03E7210197BEB6540B164332CEBED42C003ECE8DE70EB6540C99AB69AACED42C0916A8E626FEB654040F4DAAC9EED42C0615CA5D035EB6540F2B295FC50EB42C04AA3B89940EB6540D90F9D94DCEA42C00937B99972EB6540163FEB35F4E942C0B9103A5876EB65408D6D995DE5E942C008A85AD68FEB654069D2CB43DDE942C0D24A26924CEC6540C455AF6CB0EC42C0D21275304CEC6540E6CE3803B6EC42C018EA6B3714EC6540D17726991DEE42C00D91731C00EC65401BE20E8A9CEE42C0EBE45150F7EB6540D10F6A10D4EE42C01C6BD51EEDEB6540CD6886390AEF42C0FB975FA7EBEB6540DB85E63A0DEF42C0D2B47A3DAEEB65402E86A80212EF42C0",
"date_adjusted": "2011-03-25T07:30:45",
"survey_reference": None,
"adjusted_nodes": 1122,
},
"+": {
"id": 1424927,
"geom":
"01060000000100000001030000000100000012000000D2B47A3DAEEB65402E86A80212EF42C01D23796880EB6540D54A46E909EE42C03E7210197BEB6540B164332CEBED42C003ECE8DE70EB6540C99AB69AACED42C0916A8E626FEB654040F4DAAC9EED42C0615CA5D035EB6540F2B295FC50EB42C04AA3B89940EB6540D90F9D94DCEA42C00937B99972EB6540163FEB35F4E942C0B9103A5876EB65408D6D995DE5E942C008A85AD68FEB654069D2CB43DDE942C0D24A26924CEC6540C455AF6CB0EC42C0D21275304CEC6540E6CE3803B6EC42C018EA6B3714EC6540D17726991DEE42C00D91731C00EC65401BE20E8A9CEE42C0EBE45150F7EB6540D10F6A10D4EE42C01C6BD51EEDEB6540CD6886390AEF42C0FB975FA7EBEB6540DB85E63A0DEF42C0D2B47A3DAEEB65402E86A80212EF42C0",
"date_adjusted": "2011-03-25T07:30:45",
"survey_reference": "edited",
"adjusted_nodes": 1122,
},
}]
}
}
def test_init_import_name_clash(data_archive, cli_runner):
""" Import the GeoPackage into a Kart repository of the same name, and checkout a working copy of the same name. """
with data_archive("gpkg-editing") as data:
r = cli_runner.invoke(
["init", "--import", f"GPKG:editing.gpkg", "editing"])
repo_path = data / "editing"
assert r.exit_code == 0, r
assert (repo_path / ".kart" / "HEAD").exists()
repo = KartRepo(repo_path)
assert not repo.is_bare
assert not repo.is_empty
# working copy exists
wc = repo_path / "editing.gpkg"
assert wc.exists() and wc.is_file()
print("workingcopy at", wc)
assert repo.config["kart.repostructure.version"] == "3"
assert repo.config["kart.workingcopy.location"] == "editing.gpkg"
with Db_GPKG.create_engine(wc).connect() as db:
wc_rowcount = H.row_count(db, "editing")
assert wc_rowcount > 0
wc_tree_id = db.execute(
"""SELECT value FROM "gpkg_kart_state" WHERE table_name='*' AND key='tree';"""
).fetchone()[0]
assert wc_tree_id == repo.head_tree.hex
# make sure we haven't stuffed up the original file
with Db_GPKG.create_engine("editing.gpkg").connect() as dbo:
r = dbo.execute(
"SELECT 1 FROM sqlite_master WHERE name='gpkg_kart_state';")
assert not r.fetchone()
source_rowcount = dbo.execute(
"SELECT COUNT(*) FROM editing;").fetchone()[0]
assert source_rowcount == wc_rowcount
def test_feature_find_decode_performance(
profile,
archive,
source_gpkg,
table,
data_archive,
data_imported,
benchmark,
request,
):
""" Check single-feature decoding performance """
param_ids = H.parameter_ids(request)
benchmark.group = (
f"test_feature_find_decode_performance - {profile} - {param_ids[-1]}")
repo_path = data_imported(archive, source_gpkg, table)
repo = KartRepo(repo_path)
dataset = repo.datasets()["mytable"]
inner_tree = dataset.inner_tree
with data_archive(archive) as data:
with Db_GPKG.create_engine(data / source_gpkg).connect() as conn:
num_rows = conn.execute(
f"SELECT COUNT(*) FROM {table};").fetchone()[0]
pk_field = Db_GPKG.pk_name(conn, table=table)
pk = conn.execute(
f"SELECT {pk_field} FROM {table} ORDER BY {pk_field} LIMIT 1 OFFSET {min(97,num_rows-1)};"
).fetchone()[0]
if profile == "get_feature_by_pk":
benchmark(dataset.get_feature, pk)
elif profile == "get_feature_from_data":
feature_path = dataset.encode_1pk_to_path(pk, relative=True)
feature_data = memoryview(inner_tree / feature_path)
benchmark(dataset.get_feature, path=feature_path, data=feature_data)
else:
raise NotImplementedError(f"Unknown profile: {profile}")
def test_import_replace_existing_with_column_renames(
data_archive,
tmp_path,
cli_runner,
chdir,
):
with data_archive("gpkg-polygons") as data:
repo_path = tmp_path / "emptydir"
r = cli_runner.invoke(["init", repo_path])
assert r.exit_code == 0
with chdir(repo_path):
r = cli_runner.invoke([
"import",
data / "nz-waca-adjustments.gpkg",
"nz_waca_adjustments:mytable",
])
assert r.exit_code == 0, r.stderr
# Now rename
# * doesn't include the `survey_reference` column
# * has the columns in a different order
# * has a new column
with Db_GPKG.create_engine(
data / "nz-waca-adjustments.gpkg").connect() as conn:
conn.execute("""
ALTER TABLE "nz_waca_adjustments" RENAME COLUMN "survey_reference" TO "renamed_survey_reference";
""")
r = cli_runner.invoke([
"import",
"--replace-existing",
data / "nz-waca-adjustments.gpkg",
"nz_waca_adjustments:mytable",
])
assert r.exit_code == 0, r.stderr
r = cli_runner.invoke(["show", "-o", "json"])
assert r.exit_code == 0, r.stderr
diff = json.loads(r.stdout)["kart.diff/v1+hexwkb"]["mytable"]
# The schema changed, but the features didn't.
assert diff["meta"]["schema.json"]
assert not diff.get("feature")
repo = KartRepo(repo_path)
head_rs = repo.structure("HEAD")
old_rs = repo.structure("HEAD^")
assert head_rs.tree != old_rs.tree
new_feature_tree = head_rs.tree / "mytable/.table-dataset/feature"
old_feature_tree = old_rs.tree / "mytable/.table-dataset/feature"
assert new_feature_tree == old_feature_tree
def test_fetch(
data_archive_readonly,
data_working_copy,
cli_runner,
insert,
tmp_path,
request,
):
with data_working_copy("points") as (path1, wc):
subprocess.run(["git", "init", "--bare", str(tmp_path)], check=True)
r = cli_runner.invoke(["remote", "add", "myremote", tmp_path])
assert r.exit_code == 0, r
with Db_GPKG.create_engine(wc).connect() as conn:
commit_id = insert(conn)
r = cli_runner.invoke(["push", "--set-upstream", "myremote", "main"])
assert r.exit_code == 0, r
with data_working_copy("points") as (path2, wc):
repo = KartRepo(path2)
h = repo.head.target.hex
r = cli_runner.invoke(["remote", "add", "myremote", tmp_path])
assert r.exit_code == 0, r
r = cli_runner.invoke(["fetch", "myremote"])
assert r.exit_code == 0, r
H.git_graph(request, "post-fetch")
assert repo.head.name == "refs/heads/main"
assert repo.head.target.hex == h
remote_branch = repo.lookup_reference_dwim("myremote/main")
assert remote_branch.target.hex == commit_id
fetch_head = repo.lookup_reference("FETCH_HEAD")
assert fetch_head.target.hex == commit_id
# merge
r = cli_runner.invoke(["merge", "myremote/main"])
assert r.exit_code == 0, r
assert repo.head.name == "refs/heads/main"
assert repo.head.target.hex == commit_id
commit = repo.head_commit
assert len(commit.parents) == 1
assert commit.parents[0].hex == h
def print_version(ctx):
import osgeo
import psycopg2
import pysqlite3
import rtree
import sqlalchemy
from kart.sqlalchemy.gpkg import Db_GPKG
click.echo(f"Kart v{get_version()}, Copyright (c) Kart Contributors")
git_version = (subprocess.check_output(
["git", "--version"],
env=tool_environment()).decode("ascii").strip().split()[-1])
sidx_version = rtree.index.__c_api_version__.decode("ascii")
engine = Db_GPKG.create_engine(":memory:")
with engine.connect() as conn:
spatialite_version = conn.scalar("SELECT spatialite_version();")
pq_version = psycopg2.__libpq_version__
pq_version = "{}.{}.{}".format(
*
[int(k) for k in re.findall(r"\d\d", str(psycopg2.__libpq_version__))])
proj_version = "{}.{}.{}".format(
osgeo.osr.GetPROJVersionMajor(),
osgeo.osr.GetPROJVersionMinor(),
osgeo.osr.GetPROJVersionMicro(),
)
click.echo((f"» GDAL v{osgeo._gdal.__version__}; "
f"PROJ v{proj_version}\n"
f"» PyGit2 v{pygit2.__version__}; "
f"Libgit2 v{pygit2.LIBGIT2_VERSION}; "
f"Git v{git_version}\n"
f"» SQLAlchemy v{sqlalchemy.__version__}; "
f"pysqlite3 v{pysqlite3.version}/v{pysqlite3.sqlite_version}; "
f"SpatiaLite v{spatialite_version}; "
f"Libpq v{pq_version}\n"
f"» SpatialIndex v{sidx_version}"))
ctx.exit()
def _import_check(repo_path, table, source_gpkg):
repo = KartRepo(repo_path)
dataset = repo.datasets()[table]
assert dataset.VERSION == 3
with Db_GPKG.create_engine(source_gpkg).connect() as conn:
num_rows = conn.execute(f"SELECT COUNT(*) FROM {table};").fetchone()[0]
o = subprocess.check_output(["git", "ls-tree", "-r", "-t", "HEAD", table])
print("\n".join(l.decode("utf8") for l in o.splitlines()[:20]))
if dataset.VERSION != 3:
raise NotImplementedError(dataset.VERSION)
re_paths = r"^\d{6} blob [0-9a-f]{40}\t%s/.table-dataset/feature/.*$" % table
git_paths = [
m for m in re.findall(re_paths, o.decode("utf-8"), re.MULTILINE)
]
assert len(git_paths) == num_rows
num_features = dataset.feature_count
assert num_features == num_rows
return dataset
def test_status(
data_archive,
data_working_copy,
cli_runner,
insert,
tmp_path,
request,
disable_editor,
):
with data_working_copy("points") as (path1, wc):
assert text_status(cli_runner) == [
"On branch main",
"",
"Nothing to commit, working copy clean",
]
assert json_status(cli_runner) == {
"kart.status/v1": {
"commit": H.POINTS.HEAD_SHA,
"abbrevCommit": H.POINTS.HEAD_SHA[:7],
"branch": "main",
"upstream": None,
"spatialFilter": None,
"workingCopy": {
"path": str(wc),
"changes": None
},
}
}
r = cli_runner.invoke(["checkout", "HEAD~1"])
assert r.exit_code == 0, r
assert text_status(cli_runner) == [
f"HEAD detached at {H.POINTS.HEAD1_SHA[:7]}",
"",
"Nothing to commit, working copy clean",
]
assert json_status(cli_runner) == {
"kart.status/v1": {
"commit": H.POINTS.HEAD1_SHA,
"abbrevCommit": H.POINTS.HEAD1_SHA[:7],
"branch": None,
"upstream": None,
"spatialFilter": None,
"workingCopy": {
"path": str(wc),
"changes": None
},
}
}
r = cli_runner.invoke(["checkout", "main"])
assert r.exit_code == 0, r
subprocess.run(["git", "init", "--bare", str(tmp_path)], check=True)
r = cli_runner.invoke(["remote", "add", "myremote", tmp_path])
assert r.exit_code == 0, r
with Db_GPKG.create_engine(wc).connect() as db:
insert(db)
r = cli_runner.invoke(["push", "--set-upstream", "myremote", "main"])
assert r.exit_code == 0, r
assert text_status(cli_runner) == [
"On branch main",
"Your branch is up to date with 'myremote/main'.",
"",
"Nothing to commit, working copy clean",
]
jdict = json_status(cli_runner)
commit, abbrev_commit = get_commit_ids(
jdict) # This varies from run to run.
assert jdict == {
"kart.status/v1": {
"commit": commit,
"abbrevCommit": abbrev_commit,
"branch": "main",
"upstream": {
"branch": "myremote/main",
"ahead": 0,
"behind": 0,
},
"spatialFilter": None,
"workingCopy": {
"path": str(wc),
"changes": None
},
}
}
with data_working_copy("points") as (path2, wc):
engine = Db_GPKG.create_engine(wc)
r = cli_runner.invoke(["remote", "add", "myremote", tmp_path])
assert r.exit_code == 0, r
r = cli_runner.invoke(["fetch", "myremote"])
assert r.exit_code == 0, r
r = cli_runner.invoke(["branch", "--set-upstream-to=myremote/main"])
assert r.exit_code == 0, r
H.git_graph(request, "post-fetch")
assert text_status(cli_runner) == [
"On branch main",
"Your branch is behind 'myremote/main' by 1 commit, and can be fast-forwarded.",
' (use "kart pull" to update your local branch)',
"",
"Nothing to commit, working copy clean",
]
assert json_status(cli_runner) == {
"kart.status/v1": {
"commit": H.POINTS.HEAD_SHA,
"abbrevCommit": H.POINTS.HEAD_SHA[:7],
"branch": "main",
"upstream": {
"branch": "myremote/main",
"ahead": 0,
"behind": 1,
},
"spatialFilter": None,
"workingCopy": {
"path": str(wc),
"changes": None
},
}
}
# local commit
with engine.connect() as db:
insert(db, reset_index=100)
H.git_graph(request, "post-commit")
assert text_status(cli_runner) == [
"On branch main",
"Your branch and 'myremote/main' have diverged,",
"and have 1 and 1 different commits each, respectively.",
' (use "kart pull" to merge the remote branch into yours)',
"",
"Nothing to commit, working copy clean",
]
jdict = json_status(cli_runner)
commit, abbrev_commit = get_commit_ids(
jdict) # This varies from run to run.
assert jdict == {
"kart.status/v1": {
"commit": commit,
"abbrevCommit": abbrev_commit,
"branch": "main",
"upstream": {
"branch": "myremote/main",
"ahead": 1,
"behind": 1,
},
"spatialFilter": None,
"workingCopy": {
"path": str(wc),
"changes": None
},
}
}
r = cli_runner.invoke(["merge", "myremote/main"])
assert r.exit_code == 0, r
H.git_graph(request, "post-merge")
assert text_status(cli_runner) == [
"On branch main",
"Your branch is ahead of 'myremote/main' by 2 commits.",
' (use "kart push" to publish your local commits)',
"",
"Nothing to commit, working copy clean",
]
jdict = json_status(cli_runner)
commit, abbrev_commit = get_commit_ids(
jdict) # This varies from run to run.
assert jdict == {
"kart.status/v1": {
"commit": commit,
"abbrevCommit": abbrev_commit,
"branch": "main",
"upstream": {
"branch": "myremote/main",
"ahead": 2,
"behind": 0,
},
"spatialFilter": None,
"workingCopy": {
"path": str(wc),
"changes": None
},
}
}
# local edits
with engine.connect() as db:
insert(db, commit=False)
db.execute(f"DELETE FROM {H.POINTS.LAYER} WHERE fid <= 2;")
db.execute(
f"UPDATE {H.POINTS.LAYER} SET name='test0' WHERE fid <= 5;")
assert text_status(cli_runner) == [
"On branch main",
"Your branch is ahead of 'myremote/main' by 2 commits.",
' (use "kart push" to publish your local commits)',
"",
"Changes in working copy:",
' (use "kart commit" to commit)',
' (use "kart restore" to discard changes)',
"",
f" {H.POINTS.LAYER}:",
" feature:",
" 1 inserts",
" 3 updates",
" 2 deletes",
]
jdict = json_status(cli_runner)
commit, abbrev_commit = get_commit_ids(
jdict) # This varies from run to run.
assert jdict == {
"kart.status/v1": {
"commit": commit,
"abbrevCommit": abbrev_commit,
"branch": "main",
"upstream": {
"branch": "myremote/main",
"ahead": 2,
"behind": 0,
},
"spatialFilter": None,
"workingCopy": {
"path": str(wc),
"changes": {
"nz_pa_points_topo_150k": {
"feature": {
"inserts": 1,
"updates": 3,
"deletes": 2,
},
}
},
},
}
}
def test_e2e(
archive,
gpkg,
table_ref,
data_archive,
tmp_path,
chdir,
cli_runner,
insert,
request,
):
metadata = H.metadata(table_ref)
table = metadata.LAYER
row_count = metadata.ROWCOUNT
repo_path = tmp_path / "myproject"
repo_path.mkdir()
remote_path = tmp_path / "myremote"
remote_path.mkdir()
with chdir(remote_path):
# initialise empty repo for remote
subprocess.run(["git", "init", "--bare", str(remote_path)], check=True)
with data_archive(archive) as data:
with chdir(repo_path):
# initialise empty repo
r = cli_runner.invoke(["init"])
assert r.exit_code == 0
assert not (repo_path / ".sno").exists()
assert (repo_path / ".kart").is_dir()
assert (repo_path / "KART_README.txt").is_file()
# import data
r = cli_runner.invoke(["import", f"GPKG:{data / gpkg}", table])
assert r.exit_code == 0
# check there's a commit
r = cli_runner.invoke(["log"])
assert r.exit_code == 0
assert "Import from " in r.stdout
sha_import = r.stdout.splitlines()[0].split()[1]
print("Imported SHA:", sha_import)
# checkout a working copy
r = cli_runner.invoke(["checkout"])
assert r.exit_code == 0
working_copy = repo_path / "myproject.gpkg"
assert working_copy.exists()
# check we have the right data in the WC
with Db_GPKG.create_engine(working_copy).connect() as conn:
assert H.row_count(conn, table) == row_count
assert H.table_pattern_count(conn, "gpkg_kart_%") == 2
assert H.table_pattern_count(conn, "gpkg_sno_%") == 0
# create & switch to a new branch
r = cli_runner.invoke(["switch", "-c", "edit-1"])
assert r.exit_code == 0
assert r.stdout.splitlines(
)[0] == "Creating new branch 'edit-1'..."
r = cli_runner.invoke(["status"])
assert r.exit_code == 0
assert r.stdout.splitlines()[0] == "On branch edit-1"
# make an edit
with Db_GPKG.create_engine(working_copy).connect() as conn:
insert(conn, commit=False)
r = cli_runner.invoke(["diff"])
assert r.exit_code == 0
line = r.stdout.splitlines()[0]
assert re.match(fr"\+\+\+ {table}:feature:\d+$", line), line
# commit it
r = cli_runner.invoke(["commit", "-m", "commit-1"])
assert r.exit_code == 0
sha_edit1 = r.stdout.splitlines()[-1].split()[1]
print("Edit SHA:", sha_edit1)
# go back to main
r = cli_runner.invoke(["switch", "main"])
assert r.exit_code == 0
assert r.stdout.splitlines(
)[0] == f"Updating {working_copy.name} ..."
r = cli_runner.invoke(["status"])
assert r.exit_code == 0
assert r.stdout.splitlines()[0] == "On branch main"
# merge it
r = cli_runner.invoke(
["merge", "edit-1", "--no-ff", "-m", "merge-1"])
assert r.exit_code == 0
assert "Fast-forward" not in r.stdout
sha_merge1 = r.stdout.splitlines()[-1].split(" ")[-1]
print("Merge SHA:", sha_merge1)
H.git_graph(request, "post edit-1 merge", count=10)
# add a remote
r = cli_runner.invoke(["remote", "add", "myremote", remote_path])
assert r.exit_code == 0
# push
r = cli_runner.invoke(
["push", "--set-upstream", "myremote", "main"])
assert r.exit_code == 0
line = r.stdout.splitlines()[0]
patterns = [
r"[Bb]ranch '?main'? set up to track '?myremote/main'?\.$",
r"[Bb]ranch '?main'? set up to track remote branch '?main'? from '?myremote'?\.$",
]
assert any(re.match(p, line) for p in patterns), line
# check reflog
r = cli_runner.invoke(["reflog"])
assert r.exit_code == 0
assert [x.split(" ", 1)[1]
for x in r.stdout.splitlines()][0:4] == [
"HEAD@{0}: commit (merge): merge-1",
"HEAD@{1}: checkout: moving from edit-1 to main",
"HEAD@{2}: commit: commit-1",
"HEAD@{3}: checkout: moving from main to edit-1",
]
def test_import_from_non_gpkg(
archive,
source_gpkg,
table,
data_archive,
tmp_path,
cli_runner,
chdir,
request,
source_format,
source_ogr_driver,
):
"""
Import something else into a Kart repository.
"""
param_ids = H.parameter_ids(request)
with data_archive(archive) as data:
with Db_GPKG.create_engine(data / source_gpkg).connect() as conn:
if param_ids[-1] == "empty":
print(f"emptying table {table}...")
conn.execute(f"DELETE FROM {table};")
num_rows = conn.execute(
f"SELECT COUNT(*) FROM {table};").fetchone()[0]
if param_ids[-1] == "empty":
assert num_rows == 0
# First, import the original GPKG to one repo
gpkg_repo_path = tmp_path / "gpkg"
gpkg_repo_path.mkdir()
with chdir(gpkg_repo_path):
r = cli_runner.invoke(["init"])
assert r.exit_code == 0, r
r = cli_runner.invoke(["import", data / source_gpkg, table])
assert r.exit_code == 0, r
gpkg_repo = KartRepo(gpkg_repo_path)
gpkg_dataset = gpkg_repo.datasets()[table]
# convert to a new format using OGR
source_filename = tmp_path / f"data.{source_format.lower()}"
gdal.VectorTranslate(
str(source_filename),
gdal.OpenEx(str(data / source_gpkg)),
format=source_ogr_driver,
layers=[table],
)
repo_path = tmp_path / "non-gpkg"
repo_path.mkdir()
with chdir(repo_path):
r = cli_runner.invoke(["init"])
assert r.exit_code == 0, r
repo = KartRepo(repo_path)
assert repo.is_empty
# Import from SHP/TAB/something into Kart
r = cli_runner.invoke([
"import",
str(source_filename),
f"data:{table}",
])
assert r.exit_code == 0, r
assert not repo.is_empty
assert repo.head.name == "refs/heads/main"
assert repo.head.shorthand == "main"
# has a single commit
assert len([c for c in repo.walk(repo.head.target)]) == 1
dataset = _import_check(repo_path, table, f"{data / source_gpkg}")
# Compare the meta items to the GPKG-imported ones
repo = KartRepo(repo_path)
dataset = repo.datasets()[table]
_compare_ogr_and_gpkg_meta_items(dataset, gpkg_dataset)
if num_rows > 0:
# compare the first feature in the repo against the source DB
got_feature = next(dataset.features())
pk = got_feature[dataset.primary_key]
src_ds = ogr.Open(str(source_filename))
src_layer = src_ds.GetLayer(0)
assert src_layer.GetFeatureCount() == num_rows
f = src_layer.GetFeature(pk)
expected_feature = {
f.GetFieldDefnRef(i).GetName(): f.GetField(i)
for i in range(f.GetFieldCount())
}
if "date_adjus" in expected_feature:
expected_feature["date_adjus"] = expected_feature[
"date_adjus"].replace("/", "-")
expected_feature["FID"] = f.GetFID()
if src_layer.GetGeomType() != ogr.wkbNone:
g = f.GetGeometryRef()
if g:
g.AssignSpatialReference(src_layer.GetSpatialRef())
if table == H.POLYGONS.LAYER:
g = ogr.ForceToMultiPolygon(g)
expected_feature["geom"] = ogr_to_gpkg_geom(g)
assert normalise_feature(got_feature) == expected_feature
def test_import(
profile,
archive,
source_gpkg,
table,
data_archive,
tmp_path,
cli_runner,
chdir,
benchmark,
request,
monkeypatch,
):
""" Import the GeoPackage (eg. `kx-foo-layer.gpkg`) into a Kart repository. """
param_ids = H.parameter_ids(request)
# wrap the original functions with benchmarking
orig_import_func = fast_import.fast_import_tables
orig_checkout_func = init._add_datasets_to_working_copy
def _benchmark_import(*args, **kwargs):
# one round/iteration isn't very statistical, but hopefully crude idea
return benchmark.pedantic(orig_import_func,
args=args,
kwargs=kwargs,
rounds=1,
iterations=1)
def _benchmark_checkout(*args, **kwargs):
return benchmark.pedantic(orig_checkout_func,
args=args,
kwargs=kwargs,
rounds=1,
iterations=1)
if profile == "fast_import":
monkeypatch.setattr(init, "fast_import_tables", _benchmark_import)
else:
monkeypatch.setattr(init, "_add_datasets_to_working_copy",
_benchmark_checkout)
with data_archive(archive) as data:
# list tables
repo_path = tmp_path / "repo"
repo_path.mkdir()
with Db_GPKG.create_engine(data / source_gpkg).connect() as conn:
if param_ids[-1] == "empty":
print(f"emptying table {table}...")
conn.execute(f"DELETE FROM {table};")
num_rows = conn.execute(
f"SELECT COUNT(*) FROM {table};").fetchone()[0]
benchmark.group = f"test_import - {param_ids[-1]} (N={num_rows})"
if param_ids[-1] == "empty":
assert num_rows == 0
with chdir(repo_path):
r = cli_runner.invoke(["init"])
assert r.exit_code == 0, r
repo = KartRepo(repo_path)
assert repo.is_empty
r = cli_runner.invoke(["import", str(data / source_gpkg), table])
assert r.exit_code == 0, r
assert not repo.is_empty
assert repo.head.name == "refs/heads/main"
assert repo.head.shorthand == "main"
# has a single commit
assert len(list(repo.walk(repo.head.target))) == 1
dataset = _import_check(repo_path, table, f"{data / source_gpkg}")
with Db_GPKG.create_engine(data / source_gpkg).connect() as conn:
pk_field = Db_GPKG.pk_name(conn, table=table)
if num_rows > 0:
# compare the first feature in the repo against the source DB
feature = next(dataset.features())
row = normalise_feature(
conn.execute(
f"SELECT * FROM {table} WHERE {pk_field}=?;",
[feature[pk_field]],
).fetchone())
feature = normalise_feature(feature)
print("First Feature:", feature, row)
assert feature == row
# compare a source DB feature against the repo feature
row = normalise_feature(
conn.execute(
f"SELECT * FROM {table} ORDER BY {pk_field} LIMIT 1 OFFSET {min(97,num_rows-1)};"
).fetchone())
for feature in dataset.features():
if feature[pk_field] == row[pk_field]:
feature = normalise_feature(feature)
assert feature == row
break
else:
pytest.fail(
f"Couldn't find repo feature {pk_field}={row[pk_field]}"
)
def test_init_import_alt_names(data_archive, tmp_path, cli_runner, chdir):
""" Import the GeoPackage (eg. `kx-foo-layer.gpkg`) into a Kart repository. """
repo_path = tmp_path / "repo"
repo_path.mkdir()
r = cli_runner.invoke(
["init", str(repo_path), "--workingcopy-path=wc.gpkg"])
assert r.exit_code == 0, r
ARCHIVE_PATHS = (
(
"gpkg-points",
"nz-pa-points-topo-150k.gpkg",
"nz_pa_points_topo_150k",
"pa_sites",
),
(
"gpkg-polygons",
"nz-waca-adjustments.gpkg",
"nz_waca_adjustments",
"misc/waca",
),
(
"gpkg-polygons",
"nz-waca-adjustments.gpkg",
"nz_waca_adjustments",
"other/waca2",
),
)
for archive, source_gpkg, source_table, import_path in ARCHIVE_PATHS:
with data_archive(archive) as source_path:
with chdir(repo_path):
r = cli_runner.invoke([
"import",
f"GPKG:{source_path / source_gpkg}",
f"{source_table}:{import_path}",
])
assert r.exit_code == 0, r
with chdir(repo_path):
# working copy exists
with Db_GPKG.create_engine("wc.gpkg").connect() as conn:
expected_tables = set(a[3].replace("/", "__")
for a in ARCHIVE_PATHS)
r = conn.execute(
"SELECT name FROM sqlite_master WHERE type='table';")
db_tables = set(row[0] for row in r)
assert expected_tables <= db_tables
for gpkg_t in (
"gpkg_contents",
"gpkg_geometry_columns",
"gpkg_metadata_reference",
):
r = conn.execute(f"SELECT DISTINCT table_name FROM {gpkg_t};")
table_list = set(row[0] for row in r)
assert expected_tables >= table_list, gpkg_t
r = cli_runner.invoke(["diff"])
assert r.exit_code == 0, r
assert r.stdout.splitlines() == []
def test_clone(
working_copy,
data_archive,
tmp_path,
cli_runner,
chdir,
branch_name,
branch_ref,
):
with data_archive("points") as remote_path:
if branch_ref:
# add a tag
with chdir(remote_path):
subprocess.check_output(
["git", "branch", branch_name, branch_ref])
with chdir(tmp_path):
args = [
"clone",
remote_path,
("--checkout" if working_copy else "--no-checkout"),
]
if branch_ref:
args.append(f"--branch={branch_name}")
r = cli_runner.invoke(args)
repo_path = tmp_path / "points"
assert repo_path.is_dir()
r = subprocess.check_output(
["git", "-C",
str(repo_path), "config", "--local", "--list"])
print("git config file:", r.decode("utf-8").splitlines())
repo = KartRepo(repo_path)
assert not repo.is_empty
assert repo.head.name == f"refs/heads/{branch_name}"
if branch_ref == "HEAD^":
assert repo.head_commit.hex == H.POINTS.HEAD1_SHA
else:
assert repo.head_commit.hex == H.POINTS.HEAD_SHA
branch = repo.branches.local[repo.head.shorthand]
assert branch.is_head()
assert branch.upstream_name == f"refs/remotes/origin/{branch_name}"
assert len(repo.remotes) == 1
remote = repo.remotes["origin"]
assert remote.url == str(remote_path)
assert remote.fetch_refspecs == ["+refs/heads/*:refs/remotes/origin/*"]
wc = repo_path / f"{repo_path.stem}.gpkg"
if working_copy:
assert wc.exists() and wc.is_file()
table = H.POINTS.LAYER
assert repo.config["kart.repostructure.version"] == "3"
assert repo.config["kart.workingcopy.location"] == wc.name
with Db_GPKG.create_engine(wc).connect() as conn:
nrows = conn.execute(
f"SELECT COUNT(*) FROM {table};").fetchone()[0]
assert nrows > 0
wc_tree_id = conn.execute(
"""SELECT value FROM "gpkg_kart_state" WHERE table_name='*' AND key='tree';""",
).fetchone()[0]
assert wc_tree_id == repo.head_tree.hex
else:
assert not wc.exists()
def test_import_replace_ids(
data_archive,
tmp_path,
cli_runner,
chdir,
):
with data_archive("gpkg-polygons") as data:
repo_path = tmp_path / "emptydir"
r = cli_runner.invoke(["init", repo_path])
assert r.exit_code == 0
with chdir(repo_path):
# initial import
r = cli_runner.invoke([
"import",
data / "nz-waca-adjustments.gpkg",
# import 5 features
"--replace-ids",
"2501588\n4413497\n4411733\n4408774\n4408145",
"nz_waca_adjustments:mytable",
])
assert r.exit_code == 0, r.stderr
r = cli_runner.invoke(["show", "-o", "json"])
assert r.exit_code == 0, r.stderr
diff = json.loads(r.stdout)["kart.diff/v1+hexwkb"]["mytable"]
features = diff.get("feature")
assert len(features) == 5
# Now change four features; two deletes and two updates
with Db_GPKG.create_engine(
data / "nz-waca-adjustments.gpkg").connect() as conn:
conn.execute("""
DELETE FROM "nz_waca_adjustments" WHERE id IN (4413497, 4411733)
""")
conn.execute("""
UPDATE "nz_waca_adjustments" SET adjusted_nodes = adjusted_nodes + 5
WHERE id IN (4408774, 4408145)
""")
# import, but specify --replace-ids to match only one delete and one update
r = cli_runner.invoke([
"import",
"--replace-ids",
"4408774\n4413497",
data / "nz-waca-adjustments.gpkg",
"nz_waca_adjustments:mytable",
])
assert r.exit_code == 0, r.stderr
r = cli_runner.invoke(["show", "-o", "json"])
assert r.exit_code == 0, r.stderr
diff = json.loads(r.stdout)["kart.diff/v1+hexwkb"]["mytable"]
features = diff.get("feature")
# one update and one delete were performed, and the other two changes ignored.
assert len(features) == 2
assert features == [
{
"-": {
"id": 4408774,
"geom":
"01060000000100000001030000000100000010000000C885FE1E988C6540B64D609F81CC45C046DF4EB25E8C6540C15BEAE03CCC45C0F188658E208C654079F5E0A49FCB45C05857056A318C6540B96F466883CB45C04D1F1058508C6540DAE0582152CB45C0E056EB54828C6540CD7CF3110BCB45C0D93E44E98A8C6540A55E707CFFCA45C0DE793DF38D8C654046D02963FBCA45C02B069EEB928C65404AF6BEA728CB45C05DBB9EB3978C6540C9E3D8DF5ACB45C0C1CA5CBA9C8C654081C6820293CB45C0E4A03F0E9D8C6540C072BA6C98CB45C03E3F4785A48C6540B7EB364329CC45C0A51E5844A38C65409A9F40F370CC45C0204899AE9A8C6540DAB64D0C80CC45C0C885FE1E988C6540B64D609F81CC45C0",
"date_adjusted": "2016-12-15T15:59:07",
"survey_reference": None,
"adjusted_nodes": 2300,
},
"+": {
"id": 4408774,
"geom":
"01060000000100000001030000000100000010000000C885FE1E988C6540B64D609F81CC45C046DF4EB25E8C6540C15BEAE03CCC45C0F188658E208C654079F5E0A49FCB45C05857056A318C6540B96F466883CB45C04D1F1058508C6540DAE0582152CB45C0E056EB54828C6540CD7CF3110BCB45C0D93E44E98A8C6540A55E707CFFCA45C0DE793DF38D8C654046D02963FBCA45C02B069EEB928C65404AF6BEA728CB45C05DBB9EB3978C6540C9E3D8DF5ACB45C0C1CA5CBA9C8C654081C6820293CB45C0E4A03F0E9D8C6540C072BA6C98CB45C03E3F4785A48C6540B7EB364329CC45C0A51E5844A38C65409A9F40F370CC45C0204899AE9A8C6540DAB64D0C80CC45C0C885FE1E988C6540B64D609F81CC45C0",
"date_adjusted": "2016-12-15T15:59:07",
"survey_reference": None,
"adjusted_nodes": 2305,
},
},
{
"-": {
"id": 4413497,
"geom":
"0106000000010000000103000000010000000F000000A51E5844A38C65409A9F40F370CC45C03BD400EF8E8C6540D6CC24AA13CB45C0DE793DF38D8C654046D02963FBCA45C0ACBE5F719D8C6540ED43F29FDBCA45C0E6453C0ED18C6540EDF0D7648DCA45C017E20260E58C654085B9388570CA45C04CCEFA24208D65407C735A9C7ACA45C0E4045C46208D654023A031F87CCA45C082F17D01268D6540F83908FAE7CA45C090D42C9B2B8D65406A8A6F8D50CB45C0C5E452BB2C8D654067D97F9380CB45C0A54D1AC92B8D65404EFDE10287CB45C0818F66D1208D65401F20A9CF9FCB45C06E75AA0CAC8C6540C74CFD1763CC45C0A51E5844A38C65409A9F40F370CC45C0",
"date_adjusted": "2016-12-16T11:10:05",
"survey_reference": None,
"adjusted_nodes": 1296,
}
},
]
def test_import_replace_existing_with_compatible_schema_changes(
data_archive,
tmp_path,
cli_runner,
chdir,
):
with data_archive("gpkg-polygons") as data:
repo_path = tmp_path / "emptydir"
r = cli_runner.invoke(["init", repo_path])
assert r.exit_code == 0
with chdir(repo_path):
r = cli_runner.invoke([
"import",
data / "nz-waca-adjustments.gpkg",
"nz_waca_adjustments:mytable",
])
assert r.exit_code == 0, r.stderr
# Now replace with a table which
# * doesn't include the `survey_reference` column
# * has the columns in a different order
# * has a new column
with Db_GPKG.create_engine(
data / "nz-waca-adjustments.gpkg").connect() as conn:
conn.execute("""
CREATE TABLE IF NOT EXISTS "nz_waca_adjustments_2" (
"id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
"geom" MULTIPOLYGON,
"date_adjusted" DATETIME,
"adjusted_nodes" MEDIUMINT,
"newcolumn" TEXT
);
""")
conn.execute("""
INSERT INTO nz_waca_adjustments_2 (id, geom, date_adjusted, adjusted_nodes, newcolumn)
SELECT id, geom, date_adjusted, adjusted_nodes, NULL FROM nz_waca_adjustments;
""")
conn.execute("""DROP TABLE nz_waca_adjustments;""")
conn.execute(
"""ALTER TABLE "nz_waca_adjustments_2" RENAME TO "nz_waca_adjustments";"""
)
r = cli_runner.invoke([
"import",
"--replace-existing",
data / "nz-waca-adjustments.gpkg",
"nz_waca_adjustments:mytable",
])
assert r.exit_code == 0, r.stderr
r = cli_runner.invoke(["show", "-o", "json"])
assert r.exit_code == 0, r.stderr
diff = json.loads(r.stdout)["kart.diff/v1+hexwkb"]["mytable"]
# The schema changed, but the features didn't.
assert diff["meta"]["schema.json"]
assert not diff.get("feature")
repo = KartRepo(repo_path)
head_rs = repo.structure("HEAD")
old_rs = repo.structure("HEAD^")
assert head_rs.tree != old_rs.tree
new_feature_tree = head_rs.tree / "mytable/.table-dataset/feature"
old_feature_tree = old_rs.tree / "mytable/.table-dataset/feature"
assert new_feature_tree == old_feature_tree
def test_pull(
data_archive_readonly,
data_working_copy,
cli_runner,
insert,
tmp_path,
request,
chdir,
):
with data_working_copy("points") as (path1,
wc1), data_working_copy("points") as (
path2,
wc2,
):
with chdir(path1):
subprocess.run(["git", "init", "--bare",
str(tmp_path)],
check=True)
r = cli_runner.invoke(["remote", "add", "origin", tmp_path])
assert r.exit_code == 0, r
r = cli_runner.invoke(["push", "--set-upstream", "origin", "main"])
assert r.exit_code == 0, r
with chdir(path2):
r = cli_runner.invoke(["remote", "add", "origin", tmp_path])
assert r.exit_code == 0, r
r = cli_runner.invoke(["fetch", "origin"])
assert r.exit_code == 0, r
r = cli_runner.invoke(["branch", "--set-upstream-to=origin/main"])
assert r.exit_code == 0, r
with chdir(path1):
with Db_GPKG.create_engine(wc1).connect() as conn:
commit_id = insert(conn)
r = cli_runner.invoke(["push"])
assert r.exit_code == 0, r
with chdir(path2):
repo = KartRepo(path2)
h = repo.head.target.hex
r = cli_runner.invoke(["pull"])
assert r.exit_code == 0, r
H.git_graph(request, "post-pull")
remote_branch = repo.lookup_reference_dwim("origin/main")
assert remote_branch.target.hex == commit_id
assert repo.head.name == "refs/heads/main"
assert repo.head.target.hex == commit_id
commit = repo.head_commit
assert len(commit.parents) == 1
assert commit.parents[0].hex == h
# pull again / no-op
r = cli_runner.invoke(["branch", "--unset-upstream"])
assert r.exit_code == 0, r
r = cli_runner.invoke(["pull"])
assert r.exit_code == 0, r
assert repo.head.target.hex == commit_id