def get_changed_files(self, old_commit, new_commit):
"""
For each file that has changed, yields a three-tuple containing the filename, old content and new content
"""
if old_commit is not None and not self.pygit.descendant_of(
new_commit, old_commit
):
raise ValueError("Second commit must be a descendant of first commit")
old_index = pygit2.Index()
new_index = pygit2.Index()
if old_commit is not None:
old_tree = self.pygit.get(old_commit).tree
old_index.read_tree(old_tree)
else:
# This is a special hash that represents an empty tree
old_tree = self.pygit.get("4b825dc642cb6eb9a060e54bf8d69288fbee4904")
new_tree = self.pygit.get(new_commit).tree
new_index.read_tree(new_tree)
for patch in self.pygit.diff(old_tree, new_tree):
if patch.delta.status_char() != "M":
continue
if not patch.delta.new_file.path.startswith("locales/"):
continue
old_file_oid = old_index[patch.delta.old_file.path].oid
new_file_oid = new_index[patch.delta.new_file.path].oid
old_file = self.pygit.get(old_file_oid)
new_file = self.pygit.get(new_file_oid)
yield patch.delta.new_file.path, old_file.data, new_file.data
def start_session(self, name: str) -> None:
if self._current_session_index is not None:
raise Exception("Index was supposed to be None")
if self._current_session_branch is not None:
raise Exception("Session was supposed to be None")
session_branch = f"session/{name}"
# If the branch already exists, just append a unique identifier.
if session_branch in self.repo.branches:
session_branch = session_branch + "." + uuid.uuid4().hex[:7]
# Create the session branch.
self.repo.create_branch(
session_branch, self.repo.references[self.namespace_ref].peel())
# Store the session branch and index.
self._current_session_branch = session_branch
self._current_session_index = pygit2.Index()
# Populate the index with the content of the namespace branch.
namespace_ref = self.repo.references[self.namespace_ref].target
tree = self.repo[namespace_ref].tree
self._current_session_index.read_tree(tree)
def createCommits(repo, commitsData, destHead, branchName):
index = pygit2.Index()
destCommit = destHead
index.read_tree(destCommit.tree)
for commitData in commitsData:
srcCommit = commitData.commit
tree = commitData.commit.tree
for file in commitData.files:
try:
fileObject = tree[file]
index.add(
pygit2.IndexEntry(file, fileObject.id,
pygit2.GIT_FILEMODE_BLOB))
except KeyError:
removeFileFromTree(index, file)
destTree = index.write_tree(repo)
diff = repo[destTree].diff_to_tree(destCommit.tree)
if len(diff):
destCommitId = repo.create_commit('refs/heads/' + branchName,
srcCommit.author,
srcCommit.committer,
srcCommit.message, destTree,
[destCommit.id])
destCommit = repo[destCommitId]
print srcCommit.id
def setUp(self):
super().setUp()
# Create a commit that creates some files to change
index = pygit2.Index()
self.add_file_to_index(self.repo, index, "test.txt", "this is a test")
self.first_commit_id = self.make_commit_from_index(
self.repo, index, "First commit")
def test_write_feature_performance(
repo_version,
archive,
source_gpkg,
table,
data_archive,
tmp_path,
cli_runner,
chdir,
benchmark,
request,
):
""" Per-feature import performance. """
param_ids = H.parameter_ids(request)
with data_archive(archive) as data:
# list tables
repo_path = tmp_path / "data.sno"
repo_path.mkdir()
benchmark.group = f"test_write_feature_performance - {param_ids[-1]}"
with chdir(repo_path):
r = cli_runner.invoke(["init", "--repo-version", repo_version])
assert r.exit_code == 0, r
repo = pygit2.Repository(str(repo_path))
source = OgrImportSource.open(data / source_gpkg, table=table)
with source:
dataset = structure.DatasetStructure.for_version(repo_version)(
None, table)
feature_iter = itertools.cycle(source.features())
index = pygit2.Index()
if repo_version == "1":
kwargs = {
"geom_cols": source.geom_cols,
"field_cid_map": dataset.get_field_cid_map(source),
"primary_key": source.primary_key,
"cast_primary_key": False,
}
elif repo_version == "2":
kwargs = {"schema": source.schema}
def _write_feature():
feature = next(feature_iter)
dest_path, dest_data = dataset.encode_feature(
feature, **kwargs)
blob_id = repo.create_blob(dest_data)
entry = pygit2.IndexEntry(f"{dataset.path}/{dest_path}",
blob_id,
pygit2.GIT_FILEMODE_BLOB)
index.add(entry)
benchmark(_write_feature)
def workdir_diff(self):
# This is the main reason to hold onto this context throughout an entire diff -
# we can reuse this result for more than one dataset.
index = pygit2.Index(str(self.workdir_index_path))
index._repo = self.repo
return index.diff_to_workdir(
pygit2.GIT_DIFF_INCLUDE_UNTRACKED
| pygit2.GIT_DIFF_UPDATE_INDEX
# GIT_DIFF_UPDATE_INDEX just updates timestamps in the index to make the diff quicker next time
# none of the paths or hashes change, and the end result stays the same.
)
def add_from_other_repo(self,
other_repo,
committish,
src="",
dest="",
exclude=()):
"""Copies files from another repository over to this one."""
def _copy_blob(blob_id):
# Copy object over to this repo
local_id = self.repo.create_blob(other_repo[blob_id].read_raw())
# Two blobs with same content MUST have the same id in both repos
assert local_id == blob_id
commit = resolve_committish(other_repo, committish)
node = commit.tree
src = posixpath.normpath(src)
dest = posixpath.normpath(dest)
# Traverse down to the proper subtree or blob
if src != ".":
for part in src.split(os.sep):
if not isinstance(node, pygit2.Tree):
# We reached a leaf and hence can't descend further
raise KeyError(part)
node /= part
if isinstance(node, pygit2.Blob):
# Copy only a single file
if dest == ".":
# When nothing is specified as destination, keep the file's name
dest = posixpath.split(src)[1]
_copy_blob(node.id)
self.index.add(pygit2.IndexEntry(dest, node.id, node.filemode))
return
assert isinstance(node, pygit2.Tree)
# Read tree into in-memory index and then copy the entries over
index = pygit2.Index()
index.read_tree(node)
for entry in index:
# Check if file (or the directory the file is in) is in exclude list
for path in exclude:
if entry.path == path or entry.path.startswith(f"{path}/"):
break
else:
_copy_blob(entry.id)
self.index.add(
pygit2.IndexEntry(
posixpath.normpath(posixpath.join(dest, entry.path)),
entry.id,
entry.mode,
))
def setUp(self):
super().setUp()
# Create a commit that creates some files to change
index = pygit2.Index()
self.add_file_to_index(self.repo, index, "locales/test.txt",
"this is a test")
self.add_file_to_index(self.repo, index, "locales/test-change.txt",
"this is a test")
self.add_file_to_index(self.repo, index, "test-change.txt",
"this is a test")
self.add_file_to_index(self.repo, index, "locales/test-delete.txt",
"this is a test")
self.first_commit_id = self.make_commit_from_index(
self.repo, index, "First commit")
# Create a second commit with some files changed:
# test.txt remains the same
# test-change.txt was changed
# test-delete.txt was deleted
# test-added.txt was added
index = pygit2.Index()
self.add_file_to_index(self.repo, index, "locales/test.txt",
"this is a test")
self.add_file_to_index(
self.repo,
index,
"locales/test-change.txt",
"this is a test that has been changed",
)
self.add_file_to_index(self.repo, index, "test-change.txt",
"this is a test that has been changed")
self.add_file_to_index(self.repo, index, "locales/test-added.txt",
"this is a test")
self.second_commit_id = self.make_commit_from_index(
self.repo, index, "Second commit")
def test_write_feature_performance(
archive,
source_gpkg,
table,
data_archive,
tmp_path,
cli_runner,
chdir,
benchmark,
request,
):
""" Per-feature import performance. """
param_ids = H.parameter_ids(request)
with data_archive(archive) as data:
# list tables
repo_path = tmp_path / "repo"
repo_path.mkdir()
benchmark.group = f"test_write_feature_performance - {param_ids[-1]}"
with chdir(repo_path):
r = cli_runner.invoke(["init"])
assert r.exit_code == 0, r
repo = KartRepo(repo_path)
source = TableImportSource.open(data / source_gpkg, table=table)
with source:
dataset = TableV3.new_dataset_for_writing(
table, source.schema, MemoryRepo())
feature_iter = itertools.cycle(list(source.features()))
index = pygit2.Index()
encode_kwargs = {"schema": source.schema}
def _write_feature():
feature = next(feature_iter)
dest_path, dest_data = dataset.encode_feature(
feature, **encode_kwargs)
blob_id = repo.create_blob(dest_data)
entry = pygit2.IndexEntry(f"{dataset.path}/{dest_path}",
blob_id,
pygit2.GIT_FILEMODE_BLOB)
index.add(entry)
benchmark(_write_feature)
def write(self, path):
"""
Serialise this MergeIndex to the given path.
Regular entries, conflicts, and resolves are each serialised separately,
so that they can be roundtripped accurately.
"""
index = pygit2.Index(str(path))
index.clear()
for e in self.entries.values():
index.add(pygit2.IndexEntry(e.path, e.id, e.mode))
for e in self._serialise_conflicts():
index.add(pygit2.IndexEntry(e.path, e.id, e.mode))
for e in self._serialise_resolves():
index.add(pygit2.IndexEntry(e.path, e.id, e.mode))
index.write()
def __init__(self,
graph: "SakDbGraph",
name: str,
path: Path,
branch: str = "master") -> None:
super(SakDbNamespaceGit, self).__init__(graph, name)
self.repo = pygit2.init_repository(path, True)
self.namespace_branch = branch
self.namespace_ref = f"refs/heads/{branch}"
self._last_hash_for_populated_objs: str = ""
self._current_session_index: Optional[pygit2.Index] = None
self._current_session_branch: Optional[str] = None
if self.namespace_ref not in self.repo.references:
# author = pygit2.Signature("a b", "a@b")
author = self.repo.default_signature
committer = author
commit_message = "Initial commit"
index = pygit2.Index()
tid = index.write_tree(self.repo)
self.repo.create_commit(self.namespace_ref, author, committer,
commit_message, tid, [])
# Read the version and check if it is compatible with the current implementation.
try:
version = self.get_metadata("version")
except Exception:
version = None
if version is not None:
if not self._validate_version(version):
raise Exception(
f"Repo version ({version}) not supported, please update the system."
)
else:
# If no version was found, just store the current version.
if self.graph is None:
raise Exception("No graph configured")
with self.graph.session(msg="Set version"):
self.set_metadata("version", VERSION)
def rollback(self) -> None:
# Reset the session banch to the namespace branch.
if self._current_session_branch is None:
raise Exception(
"Something went wrong. The session branch is not set")
# Move the session branch back to the namespace.
branch = self.repo.branches[self.namespace_branch]
session_branch = self.repo.branches[self._current_session_branch]
session_branch.set_target(branch.target)
# Reset the index.
namespace_ref = self.repo.references[self.namespace_ref].target
tree = self.repo[namespace_ref].tree
self._current_session_index = pygit2.Index()
self._current_session_index.read_tree(tree)
def archive_head(repo, out=None):
commit = repo.get(repo.head.target)
timestamp = datetime.datetime.fromtimestamp(commit.commit_time)
tree = commit.peel(pygit2.Tree)
if out is None:
out = '%s.zip' % (commit.id.hex[:8])
if not timestamp:
timestamp = datetime.datetime.now()
index = pygit2.Index()
index.read_tree(tree)
zip_file = zipfile.ZipFile(out, 'w')
for entry in index:
info = zipfile.ZipInfo(entry.path, timestamp.timetuple())
content = repo.get(entry.id).read_raw()
zip_file.writestr(info, content)
def commit_translation(self,
repo,
resource,
translation,
modify_locale_po=None,
commit_message=None):
translation_po = translation.export_po()
if modify_locale_po:
modify_locale_po(translation_po)
index = pygit2.Index()
self.add_file_to_index(
repo,
index,
f"templates/{resource.path}.pot",
str(translation.source.export_po()),
)
self.add_file_to_index(repo, index, f"locales/fr/{resource.path}.po",
str(translation_po))
return self.make_commit_from_index(
repo, index, commit_message or "Added a translation")
def __enter__(self):
self._count += 1
if self._count != 1:
return self
self.lock.__enter__()
# First we create an empty index
self.index = pygit2.Index()
if self.ref is not None:
try:
ref = self.pygit2_repo.lookup_reference(self.ref)
except KeyError:
self.parents = []
else:
self.parents = [ref.peel().id]
if not self.initial_empty:
self.index.read_tree(ref.peel().tree)
else:
self.parents = []
return self
def test_copy_unmanaged_files(self):
# Create some files
index = pygit2.Index()
self.add_file_to_index(
self.repo,
index,
"locales/test.txt",
"this file is managed because it's in locales",
)
self.add_file_to_index(
self.repo,
index,
"templates/test.txt",
"this file is managed because it's in templates",
)
self.add_file_to_index(self.repo, index, "l10n.toml",
"this one is also managed")
self.add_file_to_index(self.repo, index, "README.md",
"this is an unmanaged file")
self.add_file_to_index(self.repo, index, "docs/foo.md",
"this is also an unmanaged file")
self.make_commit_from_index(self.repo, index, "First commit")
writer = self.repo.writer()
writer.copy_unmanaged_files(self.repo.reader())
writer.commit("Copied unmanaged files")
# Checkunmanaged files were copied
self.assert_file_in_tree(self.repo.gitpython.head.commit.tree,
"README.md")
# FIXME self.assert_file_in_tree(self.repo.gitpython.head.commit.tree, "docs/foo.md")
# Check managed files weren't
self.assert_file_not_in_tree(self.repo.gitpython.head.commit.tree,
"locales/test.txt")
self.assert_file_not_in_tree(self.repo.gitpython.head.commit.tree,
"templates/test.txt")
self.assert_file_not_in_tree(self.repo.gitpython.head.commit.tree,
"l10n.toml")
def create_tree_from_diff(self, diff, orig_tree=None):
"""
Given a tree and a diff, returns a new tree created by applying the diff.
Doesn't create any commits or modify the working copy at all.
If orig_tree is not None, the diff is applied from that tree.
Otherwise, uses the tree at the head of the repo.
"""
if orig_tree is None:
orig_tree = self.tree
git_index = pygit2.Index()
git_index.read_tree(orig_tree)
for ds in self.iter_at(orig_tree):
ds.write_index(diff[ds], git_index, self.repo)
L.info("Writing tree...")
new_tree_oid = git_index.write_tree(self.repo)
L.info(f"Tree sha: {new_tree_oid}")
return new_tree_oid
def read(cls, path):
"""Deserialise a MergeIndex from the given path."""
index = pygit2.Index(str(path))
if index.conflicts:
raise RuntimeError("pygit2.Index conflicts should be empty")
entries = {}
conflicts = {}
resolves = {}
for e in index:
if e.path.startswith(".conflicts/"):
key, conflict_part = cls._deserialise_conflict_part(e)
conflicts.setdefault(key, AncestorOursTheirs.EMPTY)
conflicts[key] |= conflict_part
elif e.path.startswith(".resolves/"):
key, resolve_part = cls._deserialise_resolve_part(e)
resolves.setdefault(key, [])
if resolve_part:
resolves[key] += [resolve_part]
else:
entries[e.path] = cls._ensure_entry(e)
return MergeIndex(entries, conflicts, resolves)
def write_resolved_tree(self, repo):
"""
Write all the merged entries and the resolved conflicts to a tree in the given repo.
Resolved conflicts will be written the same as merged entries in the resulting tree.
Only works when all conflicts are resolved.
"""
assert not self.unresolved_conflicts
index = pygit2.Index()
# Entries that were merged automatically by libgit2, often trivially:
for e in self.entries.values():
index.add(pygit2.IndexEntry(e.path, e.id, e.mode))
# libgit2 leaves entries in the main part of the index, even if they are conflicts.
# We make sure this index only contains merged entries and resolved conflicts.
index.remove_all(list(self._conflicts_paths()))
# Entries that have been explicitly selected to resolve conflicts:
for e in self._resolves_entries():
index.add(pygit2.IndexEntry(e.path, e.id, e.mode))
return index.write_tree(repo)
def commit(
self,
wcdiff,
message,
*,
author=None,
committer=None,
allow_empty=False,
):
"""
Update the repository structure and write the updated data to the tree
as a new commit, setting HEAD to the new commit.
NOTE: Doesn't update working-copy meta or tracking tables, this is the
responsibility of the caller.
"""
tree = self.tree
git_index = pygit2.Index()
git_index.read_tree(tree)
new_tree_oid = self.create_tree_from_diff(wcdiff)
L.info("Committing...")
user = self.repo.default_signature
# this will also update the ref (branch) to point to the current commit
new_commit = self.repo.create_commit(
"HEAD", # reference_name
author or user, # author
committer or user, # committer
message, # message
new_tree_oid, # tree
[self.repo.head.target], # parents
)
L.info(f"Commit: {new_commit}")
# TODO: update reflog
return new_commit
def test_bare_index(self):
index = pygit2.Index(os.path.join(self.repo.path, 'index'))
assert [x.hex for x in index] == [x.hex for x in self.repo.index]
with pytest.raises(pygit2.GitError):
index.add('bye.txt')
def commit(self, key=None, value=None, add=True, **kwargs):
"""Commit the index to the working tree.
>>> repo.add('foo', 'my very special bar')
>>> repo.commit()
>>> foo = repo.show('foo')
u'my very special bar'
If a key and value are specified, will add them immediately before
committing them.
>>> repo.commit('fast', 'and easy, too!')
>>> foo = repo.show('fast')
u'and easy, too!'
:param key: The key
:type key: string
:param value: The value of the key.
:type value: anything that runs through :func:`json.dumps`
:param author:
(optional) The signature for the author of the first commit.
Defaults to global author.
:param message:
(optional) Message for first commit. Defaults to "adding [key]" if
there was no prior value.
:type message: string
:param author:
(optional) The signature for the committer of the first commit.
Defaults to git's `--global` `author.name` and `author.email`.
:type author: pygit2.Signature
:param committer:
(optional) The signature for the committer of the first commit.
Defaults to author.
:type committer: pygit2.Signature
:param parents:
(optional) The parents of this commit. Defaults to the last commit
for this key if it already exists, or an empty list if not.
:type parents: list of :class:`Commit <jsongit.wrappers.Commit>`
:raises:
:class:`NotJsonError <jsongit.NotJsonError>`
:class:`InvalidKeyError <jsongit.InvalidKeyError>`
"""
keys = [key] if key is not None else [e.path for e in self._repo.index]
message = kwargs.pop('message', '')
parents = kwargs.pop('parents', None)
author = kwargs.pop(
'author', utils.signature(self._global_name, self._global_email))
committer = kwargs.pop('committer', author)
if kwargs:
raise TypeError("Unknown keyword args %s" % kwargs)
if key is None and value is not None:
raise InvalidKeyError()
if parents is not None:
for parent in parents:
if parent.repo != self:
raise DifferentRepoError()
if add is True and key is not None and value is not None:
self.add(key, value)
repo_head = self._repo_head()
tree_id = self._repo.index.write_tree()
self._repo.create_commit(self._head_target(), author, committer,
message, tree_id,
[repo_head.oid] if repo_head else [])
# TODO This will create some keys but not others if there is a bad key
for key in keys:
if parents is None:
parents = [self.head(key)] if self.committed(key) else []
try:
# create a single-entry tree for the commit.
blob_id = self._navigate_tree(tree_id, key)
idx = pygit2.Index('')
idx.add(
pygit2.IndexEntry(key, blob_id, pygit2.GIT_FILEMODE_BLOB))
key_tree_id = idx.write_tree(self._repo)
self._repo.create_commit(self._key2ref(key), author, committer,
message, key_tree_id,
[parent.oid for parent in parents])
except (pygit2.GitError, OSError) as e:
if (str(e).startswith('Failed to create reference')
or 'directory' in str(e)):
raise InvalidKeyError(e)
else:
raise e
def __init__(self, repo, base_committish=None):
self.repo = repo
self.index = pygit2.Index()
self.load_base(base_committish)
def convert(repo, branch, product_name, get_branch):
def add_module(module):
index.add(
pygit2.IndexEntry(module.path,
module.get().oid, pygit2.GIT_FILEMODE_COMMIT))
# Find first commit in source branch
# and create child
current = CommitPtr(branch, "", product_name)
current.index = len(current.list) - 1
heap = [current]
parents = []
while heap:
current = heap[0]
print(product_name,
datetime.datetime.utcfromtimestamp(current.get().committer.time))
if current.path == "":
parents.append(current.get().oid)
heap = []
index = pygit2.Index()
index.read_tree(repo[current.get().tree_id])
# Read modules
blob_id = index[".gitmodules"].id
text = repo[blob_id].read_raw().decode("latin1")
modules = ""
for name, path, tag, url in sorted(
x.groups() for x in __parse_modules_re.finditer(text)):
# Load tag or branch
if tag is None:
tag = "master"
module = CommitPtr(get_branch(name, url, tag), path,
posixpath.basename(path))
# Find latest commit before current
while module.index < len(module.list) and module > current:
module.index += 1
if module.index < len(module.list):
add_module(module)
# Add symbolic links for moved components
link = path_map.get(module.path)
if link is not None and current.get(
).committer.time < rearrange_date:
index.add(
pygit2.IndexEntry(
link,
repo.create_blob(
posixpath.relpath(module.path,
posixpath.dirname(link))),
pygit2.GIT_FILEMODE_LINK))
module.index -= 1
if module.index >= 0:
heapq.heappush(heap, module)
else: # current_path == ""
heapq.heappop(heap)
add_module(current)
parents = [
repo.create_commit(None,
current.get().author,
current.get().committer,
current.name + ": " + current.get().message,
index.write_tree(repo), parents)
]
current.index -= 1
if current.index >= 0:
heapq.heappush(heap, current)
print(product_name + " conversion finished")
return parents[0]
args = parse_args()
repo = git.Repository('.')
# FIXME: This assumes a ref pointing to a commit
oid = repo.lookup_reference(args.treeish).resolve().target
commit = repo[oid]
timestamp = commit.committer.time
tree = commit.tree
filename = args.output
if filename == None:
filename = "%s.tar" % (args.treeish)
out = tarfile.open(filename, mode='w')
index = git.Index()
index.read_tree(tree)
for entry in index:
content = repo[entry.id].read_raw()
info = tarfile.TarInfo(entry.name)
info.size = len(content)
info.mtime = timestamp
info.uname = info.gname = 'root' # just because git does this
if entry.mode == SYMLINK:
info.type = tarfile.SYMTYPE
info.linkname = content
info.mode = 0777 # symlinks get placeholder
tar.addfile(info, StringIO(content))
def upgrade(source, dest, layer):
"""
Upgrade a v0.0/v0.1 Sno repository to Sno v0.2
"""
source = Path(source)
dest = Path(dest)
if dest.exists():
raise click.BadParameter(f"'{dest}': already exists",
param_hint="DEST")
source_repo = pygit2.Repository(str(source))
if not source_repo or not source_repo.is_bare:
raise click.BadParameter(f"'{source}': not an existing repository",
param_hint="SOURCE")
try:
source_tree = source_repo.head.peel(pygit2.Tree) / layer
except KeyError:
raise click.BadParameter(f"'{layer}' not found in source repository",
param_hint="SOURCE")
try:
version_data = json.loads((source_tree / "meta" / "version").data)
version = tuple([int(v) for v in version_data["version"].split(".")])
except Exception:
raise click.BadParameter("Error getting source repository version",
param_hint="SOURCE")
if version >= (0, 2):
raise click.BadParameter(
f"Expecting version <0.2, got {version_data['version']}",
param_hint="SOURCE",
)
# action!
click.secho(f"Initialising {dest} ...", bold=True)
dest.mkdir()
dest_repo = pygit2.init_repository(str(dest), bare=True)
# walk _all_ references
source_walker = source_repo.walk(
source_repo.head.target,
pygit2.GIT_SORT_TOPOLOGICAL | pygit2.GIT_SORT_REVERSE)
for ref in source_repo.listall_reference_objects():
source_walker.push(ref.resolve().target)
commit_map = {}
click.secho("\nWriting new commits ...", bold=True)
for i, source_commit in enumerate(source_walker):
dest_parents = []
for parent_id in source_commit.parent_ids:
try:
dest_parents.append(commit_map[parent_id.hex])
except KeyError:
raise ValueError(
f"Commit {i} ({source_commit.id}): Haven't seen parent ({parent_id})"
)
source_tree = source_commit.peel(pygit2.Tree) / layer
sqlite_table_info = json.loads(
(source_tree / "meta" / "sqlite_table_info").data.decode("utf8"))
field_cid_map = {r["name"]: r["cid"] for r in sqlite_table_info}
gpkg_geometry_columns = json.loads(
(source_tree / "meta" /
"gpkg_geometry_columns").data.decode("utf8"))
geom_field = (gpkg_geometry_columns["column_name"]
if gpkg_geometry_columns else None)
pk_field = None
for field in sqlite_table_info:
if field["pk"]:
pk_field = field["name"]
break
else:
if sqlite_table_info[0]["type"] == "INTEGER":
pk_field = sqlite_table_info[0]["name"]
else:
raise ValueError("No primary key field found")
if i == 0:
click.echo(
f" {layer}: Geometry={geom_field} PrimaryKey={pk_field}")
dataset = Dataset1(None, layer)
version = json.dumps(Dataset1.VERSION_CONTENTS).encode("utf8")
feature_count = 0
index = pygit2.Index()
for top_tree, top_path, subtree_names, blob_names in walk_tree(
source_tree):
if top_path == "meta":
# copy meta across as-is
for blob_name in blob_names:
if blob_name == "version":
# except version which we update
dest_blob = dest_repo.create_blob(version)
else:
source_blob = top_tree / blob_name
dest_blob = dest_repo.create_blob(source_blob.data)
index.add(
pygit2.IndexEntry(
f"{layer}/.sno-table/{top_path}/{blob_name}",
dest_blob,
pygit2.GIT_FILEMODE_BLOB,
))
for col, cid in field_cid_map.items():
dest_blob = dest_repo.create_blob(json.dumps(cid))
index.add(
pygit2.IndexEntry(
f"{layer}/.sno-table/meta/fields/{col}",
dest_blob,
pygit2.GIT_FILEMODE_BLOB,
))
dest_blob = dest_repo.create_blob(json.dumps(pk_field))
index.add(
pygit2.IndexEntry(
f"{layer}/.sno-table/meta/primary_key",
dest_blob,
pygit2.GIT_FILEMODE_BLOB,
))
elif re.match(
r"^features/[a-f0-9]{4}/([a-f0-9]{8}-(?:[a-f0-9]{4}-){3}[a-f0-9]{12})$",
top_path,
):
# feature path
source_feature_dict = {}
for attr in blob_names:
source_blob = top_tree / attr
if attr == geom_field:
source_feature_dict[attr] = source_blob.data
else:
source_feature_dict[attr] = json.loads(
source_blob.data.decode("utf8"))
kwargs = {
"field_cid_map": field_cid_map,
"geom_cols": [geom_field],
"primary_key": pk_field,
"cast_primary_key": False,
}
dest_path, dest_data = dataset.encode_feature(
source_feature_dict, **kwargs)
blob_id = dest_repo.create_blob(dest_data)
entry = pygit2.IndexEntry(dest_path, blob_id,
pygit2.GIT_FILEMODE_BLOB)
index.add(entry)
feature_count += 1
elif top_path == "" or re.match(r"^features(/[a-f0-9]{4})?$",
top_path):
pass
else:
raise ValueError(f"Unexpected path: '{top_path}'")
dest_tree = index.write_tree(dest_repo)
dest_commit = dest_repo.create_commit(
"HEAD",
source_commit.author,
source_commit.committer,
source_commit.message,
dest_tree,
dest_parents,
# source_commit.message_encoding,
)
commit_map[source_commit.hex] = dest_commit.hex
commit_time = datetime.fromtimestamp(source_commit.commit_time)
click.echo(
f" {i}: {source_commit.hex[:8]} → {dest_commit.hex[:8]} ({commit_time}; {source_commit.committer.name}; {feature_count} rows)"
)
click.echo(f"{i+1} commits processed.")
click.secho("\nUpdating references ...", bold=True)
for ref in source_repo.listall_reference_objects():
if ref.type == pygit2.GIT_REF_OID:
# real references
target = commit_map[ref.target.hex]
dest_repo.references.create(ref.name, target, True) # overwrite
click.echo(f" {ref.name} ({ref.target.hex[:8]} → {target[:8]})")
for ref in source_repo.listall_reference_objects():
if ref.type == pygit2.GIT_REF_SYMBOLIC:
dest_repo.references.create(ref.name, ref.target)
click.echo(f" {ref.name} → {ref.target}")
click.secho("\nCompacting repository ...", bold=True)
subprocess.check_call(["git", "-C", str(dest), "gc"])
click.secho("\nUpgrade complete", fg="green", bold=True)
def test_bare_index(self):
index = pygit2.Index(os.path.join(self.repo.path, 'index'))
self.assertEqual([x.hex for x in index],
[x.hex for x in self.repo.index])
self.assertRaises(pygit2.GitError, lambda: index.add('bye.txt'))
def __init__(self, repo, repo_is_empty):
self.repo = repo
self.repo_is_empty = repo_is_empty
self.index = pygit2.Index()
def __init__(self, repo):
self.repo = repo
self.index = pygit2.Index()
self.last_commit = self.repo.head.target
self.index.read_tree(self.repo.get(self.last_commit).tree)
