def tree_changes(
self,
source,
target,
want_unchanged=False,
include_trees=False,
change_type_same=False,
rename_detector=None,
):
"""Find the differences between the contents of two trees
Args:
source: SHA1 of the source tree
target: SHA1 of the target tree
want_unchanged: Whether unchanged files should be reported
include_trees: Whether to include trees
change_type_same: Whether to report files changing
type in the same entry.
Returns: Iterator over tuples with
(oldpath, newpath), (oldmode, newmode), (oldsha, newsha)
"""
for change in tree_changes(
self,
source,
target,
want_unchanged=want_unchanged,
include_trees=include_trees,
change_type_same=change_type_same,
rename_detector=rename_detector,
):
yield (
(change.old.path, change.new.path),
(change.old.mode, change.new.mode),
(change.old.sha, change.new.sha),
)
def diff(self, old_sha, new_sha=None):
"""Show the changed files between OLD_SHA and NEW_SHA
If NEW_SHA is not set, it will default to HEAD. The output is a
list of tuples (action, filename)
:param old_sha: parent commit's sha
:param new_sha: another sha, defaults to HEAD
:retval: dict
"""
orig = self._get_object(ROOT_PATH, commit_sha=old_sha)
new = self._get_object(ROOT_PATH)
if new_sha:
new = self._get_object(ROOT_PATH, commit_sha=new_sha)
keys = {
diff_tree.CHANGE_DELETE: 'delete',
diff_tree.CHANGE_ADD: 'add',
diff_tree.CHANGE_MODIFY: 'modify'
}
out = defaultdict(list)
for change_tree in diff_tree.tree_changes(self.repo.object_store,
orig.id,
new.id,
want_unchanged=False):
if change_tree.type.lower() == "delete" and change_tree.old.path:
# if the change was a delete, we have no tree or blob to yield so return key with no value
# return in the same type of structure for consistency
out[change_tree.type].append([(change_tree.old.path, None)])
else:
out[change_tree.type].append(
filter(None, self.entries(change_tree.new.path)))
return out
def commit(self, message='Saving multiple files'):
working_tree = self._working_tree
if working_tree == self._tree:
return
if self._commit is None:
self.repo = repo = Repoman.create_repo(self.name, self.author)
self._commit = repo.last_commit
fake_store = {
self._tree.id: self._tree,
working_tree.id: working_tree,
}
blobs = []
for change in tree_changes(fake_store, self._tree.id, working_tree.id):
if change.new.sha in self._blobs:
blobs.append(self._blobs[change.new.sha])
commit = self.repo._create_commit()
commit.parents = [self._commit.id]
commit.tree = working_tree.id
commit.message = message
self.repo._update_store(commit, working_tree, *blobs)
self.repo._advance_branch(self.branch, commit)
self._commit = commit
self._tree = working_tree
self._working_tree = working_tree.copy()
def dulwichCommit(self, filePath, fullPath, kind):
git = Repo(AUTOGIT_PATH)
staged = map(str, [filePath])
git.stage(staged)
index = git.open_index()
try:
committer = git._get_user_identity()
except ValueError:
committer = "autogit"
try:
head = git.head()
except KeyError:
return git.do_commit('%s - autogit commit (via dulwich)' % kind,
committer=committer)
changes = list(
tree_changes(git, index.commit(git.object_store),
git['HEAD'].tree))
if changes and len(changes) > 0:
return git.do_commit('%s - autogit commit (via dulwich)' % kind,
committer=committer)
return None
def diff(self, old_sha, new_sha=None):
"""Show the changed files between OLD_SHA and NEW_SHA
If NEW_SHA is not set, it will default to HEAD. The output is a
list of tuples (action, filename)
:param old_sha: parent commit's sha
:param new_sha: another sha, defaults to HEAD
:retval: dict
"""
orig = self._get_object(ROOT_PATH, commit_sha=old_sha)
new = self._get_object(ROOT_PATH)
if new_sha:
new = self._get_object(ROOT_PATH, commit_sha=new_sha)
keys = { diff_tree.CHANGE_DELETE: 'delete',
diff_tree.CHANGE_ADD: 'add',
diff_tree.CHANGE_MODIFY: 'modify'}
out = defaultdict(list)
for change_tree in diff_tree.tree_changes(self.repo.object_store, orig.id, new.id, want_unchanged=False):
if change_tree.type.lower() == "delete" and change_tree.old.path:
# if the change was a delete, we have no tree or blob to yield so return key with no value
# return in the same type of structure for consistency
out[change_tree.type].append([(change_tree.old.path, None)])
else:
out[change_tree.type].append(filter(None, self.entries(change_tree.new.path)))
return out
def get_branch_changed_files(self, branch_name):
'''Returns the name of all changed files within the branch.'''
master_tree = self._get_tree(self.get_branch('master'))
branch_tree = self._get_tree(self.get_branch(branch_name))
changes = tree_changes(self._repo.object_store, branch_tree.id,
master_tree.id)
return [entry.new.path or entry.old.path for entry in changes]
def merge(self, source_branch, target_branch='master', author=None, committer=None):
if source_branch == target_branch:
raise ValueError("Cannot merge branch with itself %s" % source_branch)
target_tree = self._get_object(ROOT_PATH, target_branch)
branch_tree = self._get_object(ROOT_PATH, source_branch)
for tc in diff_tree.tree_changes(self.repo.object_store, target_tree.id, branch_tree.id):
if tc.type == diff_tree.CHANGE_ADD:
self._add_tree(target_tree, ((tc.new.path, tc.new.sha, tc.new.mode),))
if tc.type == diff_tree.CHANGE_COPY:
pass
if tc.type == diff_tree.CHANGE_DELETE:
target_tree = self._delete(tc.old.path, target_branch)
if tc.type == diff_tree.CHANGE_MODIFY:
self._add_tree(target_tree, ((tc.new.path, tc.new.sha, tc.new.mode),))
if tc.type == diff_tree.CHANGE_RENAME:
pass
if tc.type == diff_tree.CHANGE_UNCHANGED:
pass
msg = "Merge %s to %s" % (source_branch, target_branch)
merge_heads = [self.branch_head(source_branch)]
sha = self.repo.do_commit(
tree=target_tree.id,
message=msg,
ref=self._branch_ref_name(target_branch),
merge_heads=merge_heads,
author=author,
committer=committer
)
return {'sha': sha}
def get_branch_changed_files(self, branch_name):
'''Returns the name of all changed files within the branch.'''
master_tree = self._get_tree(self.get_branch('master'))
branch_tree = self._get_tree(self.get_branch(branch_name))
changes = tree_changes(
self._repo.object_store, branch_tree.id, master_tree.id)
return [entry.new.path or entry.old.path for entry in changes]
def getCatSource(self,file, rev):
if file is not None and isinstance(file, str):
file = file.encode()
if rev is not None and isinstance(rev, str):
rev = rev.encode()
commit = self.r.get_object(rev)
prev = None
tr = None
if len(commit.parents)>0 and len(commit.parents)<2:
prev = self.r.get_object(commit.parents[0])
if prev is not None:
tr = prev.tree
delta = list(diff_tree.tree_changes(self.r, tr, commit.tree))
for x in delta:
if x.type == 'add' or x.type == 'modify':
if x.new.path == file:
return Helpers.ConvertToUTF8(self.r.get_object(x.new.sha).data)
else:
if x.old.path == file and x.new.path == None:
return ""
return None
def test_hard_head(self):
f = open(os.path.join(self.repo.path, 'foo'), 'w')
try:
f.write("BAR")
finally:
f.close()
porcelain.add(self.repo.path, paths=["foo"])
porcelain.commit(self.repo.path,
message="Some message",
committer="Jane <*****@*****.**>",
author="John <*****@*****.**>")
f = open(os.path.join(self.repo.path, 'foo'), 'w')
try:
f.write("OOH")
finally:
f.close()
porcelain.reset(self.repo, "hard", "HEAD")
index = self.repo.open_index()
changes = list(
tree_changes(self.repo, index.commit(self.repo.object_store),
self.repo['HEAD'].tree))
self.assertEquals([], changes)
def test_hard_commit(self):
with open(os.path.join(self.repo.path, "foo"), "w") as f:
f.write("BAR")
porcelain.add(self.repo.path, paths=["foo"])
sha = porcelain.commit(
self.repo.path,
message=b"Some message",
committer=b"Jane <*****@*****.**>",
author=b"John <*****@*****.**>",
)
with open(os.path.join(self.repo.path, "foo"), "wb") as f:
f.write(b"BAZ")
porcelain.add(self.repo.path, paths=["foo"])
porcelain.commit(
self.repo.path,
message=b"Some other message",
committer=b"Jane <*****@*****.**>",
author=b"John <*****@*****.**>",
)
porcelain.reset(self.repo, "hard", sha)
index = self.repo.open_index()
changes = list(tree_changes(self.repo, index.commit(self.repo.object_store), self.repo[sha].tree))
self.assertEqual([], changes)
def commit(self, message='Saving multiple files'):
working_tree = self._working_tree
if working_tree == self._tree:
return
if self._commit is None:
self.repo = repo = Repoman.create_repo(self.name, self.author)
self._commit = repo.last_commit
fake_store = {
self._tree.id: self._tree,
working_tree.id: working_tree,
}
blobs = []
for change in tree_changes(fake_store, self._tree.id, working_tree.id):
if change.new.sha in self._blobs:
blobs.append(self._blobs[change.new.sha])
commit = self.repo._create_commit()
commit.parents = [self._commit.id]
commit.tree = working_tree.id
commit.message = message
self.repo._update_store(commit, working_tree, *blobs)
self.repo._advance_branch(self.branch, commit)
self._commit = commit
self._tree = working_tree
self._working_tree = working_tree.copy()
def test_hard_head(self):
f = open(os.path.join(self.repo.path, 'foo'), 'w')
try:
f.write("BAR")
finally:
f.close()
porcelain.add(self.repo.path, paths=["foo"])
porcelain.commit(self.repo.path, message="Some message",
committer="Jane <*****@*****.**>",
author="John <*****@*****.**>")
f = open(os.path.join(self.repo.path, 'foo'), 'w')
try:
f.write("OOH")
finally:
f.close()
porcelain.reset(self.repo, "hard", "HEAD")
index = self.repo.open_index()
changes = list(tree_changes(self.repo,
index.commit(self.repo.object_store),
self.repo['HEAD'].tree))
self.assertEqual([], changes)
def merge(self,
source_branch,
target_branch='master',
author=None,
committer=None):
if source_branch == target_branch:
raise ValueError("Cannot merge branch with itself %s" %
source_branch)
target_tree = self._get_object(ROOT_PATH, target_branch)
branch_tree = self._get_object(ROOT_PATH, source_branch)
for tc in diff_tree.tree_changes(self.repo.object_store,
target_tree.id, branch_tree.id):
if tc.type == diff_tree.CHANGE_ADD:
self._add_tree(target_tree,
((tc.new.path, tc.new.sha, tc.new.mode), ))
if tc.type == diff_tree.CHANGE_COPY:
pass
if tc.type == diff_tree.CHANGE_DELETE:
target_tree = self._delete(tc.old.path, target_branch)
if tc.type == diff_tree.CHANGE_MODIFY:
self._add_tree(target_tree,
((tc.new.path, tc.new.sha, tc.new.mode), ))
if tc.type == diff_tree.CHANGE_RENAME:
pass
if tc.type == diff_tree.CHANGE_UNCHANGED:
pass
msg = "Merge %s to %s" % (source_branch, target_branch)
merge_heads = [self.branch_head(source_branch)]
sha = self.repo.do_commit(tree=target_tree.id,
message=msg,
ref=self._branch_ref_name(target_branch),
merge_heads=merge_heads,
author=author,
committer=committer)
return {'sha': sha}
def test_hard_commit(self):
fullpath = os.path.join(self.repo.path, 'foo')
with open(fullpath, 'w') as f:
f.write("BAR")
porcelain.add(self.repo.path, paths=[fullpath])
sha = porcelain.commit(self.repo.path,
message=b"Some message",
committer=b"Jane <*****@*****.**>",
author=b"John <*****@*****.**>")
with open(fullpath, 'wb') as f:
f.write(b"BAZ")
porcelain.add(self.repo.path, paths=[fullpath])
porcelain.commit(self.repo.path,
message=b"Some other message",
committer=b"Jane <*****@*****.**>",
author=b"John <*****@*****.**>")
porcelain.reset(self.repo, "hard", sha)
index = self.repo.open_index()
changes = list(
tree_changes(self.repo, index.commit(self.repo.object_store),
self.repo[sha].tree))
self.assertEqual([], changes)
def get_commit_file_diffs(repo_path, max_commit_count=-1):
repo = Repo(repo_path)
prev = None
walker = repo.get_graph_walker()
commit_changes = []
commit_count = 0
cset = walker.next()
while cset is not None:
commit = repo.get_object(cset)
if prev is None:
prev = commit.tree
cset = walker.next()
continue
this_commit_changes = []
for x in tree_changes(repo, prev, commit.tree):
if x.old.path is not None:
this_commit_changes.append(x.old.path)
commit_changes.append(this_commit_changes)
prev = commit.tree
commit_count += 1
if max_commit_count > 0 and commit_count >= max_commit_count:
cset = None
else:
cset = walker.next()
return RepoDiffResult(repo_path, commit_changes, commit_count)
def _dulwich_status(self):
"""
Return the git status
"""
_repo = Repo(self.config['top_dir'])
index = _repo.open_index()
return list(tree_changes(_repo, index.commit(_repo.object_store),
_repo['HEAD'].tree))
def _dulwich_status(self):
"""
Return the git status
"""
_repo = Repo(self.config['top_dir'])
index = _repo.open_index()
return list(
tree_changes(_repo, index.commit(_repo.object_store),
_repo['HEAD'].tree))
def test_simple(self):
"""
Basic test of porcelain push where self.repo is the remote. First
clone the remote, commit a file to the clone, then push the changes
back to the remote.
"""
outstream = BytesIO()
errstream = BytesIO()
porcelain.commit(repo=self.repo.path, message=b'init',
author=b'author <email>',
committer=b'committer <email>')
# Setup target repo cloned from temp test repo
clone_path = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, clone_path)
target_repo = porcelain.clone(self.repo.path, target=clone_path,
errstream=errstream)
try:
self.assertEqual(target_repo[b'HEAD'], self.repo[b'HEAD'])
finally:
target_repo.close()
# create a second file to be pushed back to origin
handle, fullpath = tempfile.mkstemp(dir=clone_path)
os.close(handle)
porcelain.add(repo=clone_path, paths=[fullpath])
porcelain.commit(repo=clone_path, message=b'push',
author=b'author <email>',
committer=b'committer <email>')
# Setup a non-checked out branch in the remote
refs_path = b"refs/heads/foo"
new_id = self.repo[b'HEAD'].id
self.assertNotEqual(new_id, ZERO_SHA)
self.repo.refs[refs_path] = new_id
# Push to the remote
porcelain.push(clone_path, self.repo.path, b"HEAD:" + refs_path,
outstream=outstream, errstream=errstream)
# Check that the target and source
with Repo(clone_path) as r_clone:
self.assertEqual({
b'HEAD': new_id,
b'refs/heads/foo': r_clone[b'HEAD'].id,
b'refs/heads/master': new_id,
}, self.repo.get_refs())
self.assertEqual(r_clone[b'HEAD'].id, self.repo[refs_path].id)
# Get the change in the target repo corresponding to the add
# this will be in the foo branch.
change = list(tree_changes(self.repo, self.repo[b'HEAD'].tree,
self.repo[b'refs/heads/foo'].tree))[0]
self.assertEqual(os.path.basename(fullpath),
change.new.path.decode('ascii'))
def test_simple(self):
"""
Basic test of porcelain push where self.repo is the remote. First
clone the remote, commit a file to the clone, then push the changes
back to the remote.
"""
outstream = BytesIO()
errstream = BytesIO()
porcelain.commit(repo=self.repo.path, message=b'init',
author=b'author <email>',
committer=b'committer <email>')
# Setup target repo cloned from temp test repo
clone_path = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, clone_path)
target_repo = porcelain.clone(self.repo.path, target=clone_path,
errstream=errstream)
try:
self.assertEqual(target_repo[b'HEAD'], self.repo[b'HEAD'])
finally:
target_repo.close()
# create a second file to be pushed back to origin
handle, fullpath = tempfile.mkstemp(dir=clone_path)
os.close(handle)
porcelain.add(repo=clone_path, paths=[fullpath])
porcelain.commit(repo=clone_path, message=b'push',
author=b'author <email>',
committer=b'committer <email>')
# Setup a non-checked out branch in the remote
refs_path = b"refs/heads/foo"
new_id = self.repo[b'HEAD'].id
self.assertNotEqual(new_id, ZERO_SHA)
self.repo.refs[refs_path] = new_id
# Push to the remote
porcelain.push(clone_path, self.repo.path, b"HEAD:" + refs_path,
outstream=outstream, errstream=errstream)
# Check that the target and source
with Repo(clone_path) as r_clone:
self.assertEqual({
b'HEAD': new_id,
b'refs/heads/foo': r_clone[b'HEAD'].id,
b'refs/heads/master': new_id,
}, self.repo.get_refs())
self.assertEqual(r_clone[b'HEAD'].id, self.repo[refs_path].id)
# Get the change in the target repo corresponding to the add
# this will be in the foo branch.
change = list(tree_changes(self.repo, self.repo[b'HEAD'].tree,
self.repo[b'refs/heads/foo'].tree))[0]
self.assertEqual(os.path.basename(fullpath),
change.new.path.decode('ascii'))
def tree_changes(self, source, target, want_unchanged=False):
"""Find the differences between the contents of two trees
:param source: SHA1 of the source tree
:param target: SHA1 of the target tree
:param want_unchanged: Whether unchanged files should be reported
:return: Iterator over tuples with
(oldpath, newpath), (oldmode, newmode), (oldsha, newsha)
"""
for change in tree_changes(self, source, target,
want_unchanged=want_unchanged):
yield ((change.old.path, change.new.path),
(change.old.mode, change.new.mode),
(change.old.sha, change.new.sha))
def test_simple(self):
"""
Basic test of porcelain push where self.repo is the remote. First
clone the remote, commit a file to the clone, then push the changes
back to the remote.
"""
outstream = BytesIO()
errstream = BytesIO()
porcelain.commit(repo=self.repo.path,
message=b'init',
author=b'',
committer=b'')
# Setup target repo cloned from temp test repo
clone_path = tempfile.mkdtemp()
porcelain.clone(self.repo.path, target=clone_path, errstream=errstream)
# create a second file to be pushed back to origin
handle, fullpath = tempfile.mkstemp(dir=clone_path)
os.close(handle)
porcelain.add(repo=clone_path, paths=[os.path.basename(fullpath)])
porcelain.commit(repo=clone_path,
message=b'push',
author=b'',
committer=b'')
# Setup a non-checked out branch in the remote
refs_path = b"refs/heads/foo"
self.repo[refs_path] = self.repo[b'HEAD']
# Push to the remote
porcelain.push(clone_path,
self.repo.path,
refs_path,
outstream=outstream,
errstream=errstream)
# Check that the target and source
r_clone = Repo(clone_path)
# Get the change in the target repo corresponding to the add
# this will be in the foo branch.
change = list(
tree_changes(self.repo, self.repo[b'HEAD'].tree,
self.repo[b'refs/heads/foo'].tree))[0]
self.assertEqual(r_clone[b'HEAD'].id, self.repo[refs_path].id)
self.assertEqual(os.path.basename(fullpath),
change.new.path.decode('ascii'))
def tree_changes(self, source, target, want_unchanged=False):
"""Find the differences between the contents of two trees
:param source: SHA1 of the source tree
:param target: SHA1 of the target tree
:param want_unchanged: Whether unchanged files should be reported
:return: Iterator over tuples with
(oldpath, newpath), (oldmode, newmode), (oldsha, newsha)
"""
for change in tree_changes(self, source, target,
want_unchanged=want_unchanged):
yield ((change.old.path, change.new.path),
(change.old.mode, change.new.mode),
(change.old.sha, change.new.sha))
def summary(self):
if not self._summary:
if not self._a:
a_tree = DulwichTree()
else:
a_tree = self._a._commit.tree
self._summary = tree_changes(
self.repository._repo.object_store,
a_tree,
self._b._commit.tree,
rename_detector=self._rename_detector,
)
return self._summary
def getRevisionFiles(self, rev):
if rev is not None and isinstance(rev, str):
rev = rev.encode()
commit = self.r.get_object(rev)
delta = diff_tree.tree_changes(self.r, None, commit.tree)
files = []
for x in delta:
if x.type == 'add' or x.type == 'modify':
files.append(x.new.path)
return sorted(files)
def test_hard_head(self):
with open(os.path.join(self.repo.path, 'foo'), 'w') as f:
f.write("BAR")
porcelain.add(self.repo.path, paths=["foo"])
porcelain.commit(self.repo.path, message=b"Some message",
committer=b"Jane <*****@*****.**>",
author=b"John <*****@*****.**>")
with open(os.path.join(self.repo.path, 'foo'), 'wb') as f:
f.write(b"OOH")
porcelain.reset(self.repo, "hard", b"HEAD")
index = self.repo.open_index()
changes = list(tree_changes(self.repo,
index.commit(self.repo.object_store),
self.repo[b'HEAD'].tree))
self.assertEqual([], changes)
def test_simple(self):
"""
Basic test of porcelain push where self.repo is the remote. First
clone the remote, commit a file to the clone, then push the changes
back to the remote.
"""
outstream = BytesIO()
errstream = BytesIO()
porcelain.commit(repo=self.repo.path, message=b'init',
author=b'', committer=b'')
# Setup target repo cloned from temp test repo
clone_path = tempfile.mkdtemp()
target_repo = porcelain.clone(self.repo.path, target=clone_path, errstream=errstream)
target_repo.close()
# create a second file to be pushed back to origin
handle, fullpath = tempfile.mkstemp(dir=clone_path)
os.close(handle)
porcelain.add(repo=clone_path, paths=[os.path.basename(fullpath)])
porcelain.commit(repo=clone_path, message=b'push',
author=b'', committer=b'')
# Setup a non-checked out branch in the remote
refs_path = b"refs/heads/foo"
self.repo[refs_path] = self.repo[b'HEAD']
# Push to the remote
porcelain.push(clone_path, self.repo.path, refs_path, outstream=outstream,
errstream=errstream)
# Check that the target and source
with closing(Repo(clone_path)) as r_clone:
# Get the change in the target repo corresponding to the add
# this will be in the foo branch.
change = list(tree_changes(self.repo, self.repo[b'HEAD'].tree,
self.repo[b'refs/heads/foo'].tree))[0]
self.assertEqual(r_clone[b'HEAD'].id, self.repo[refs_path].id)
self.assertEqual(os.path.basename(fullpath), change.new.path.decode('ascii'))
def changes_between_git_tree_and_working_copy(source_store,
from_tree_sha,
target,
want_unchanged=False,
want_unversioned=False,
rename_detector=None,
include_trees=True):
"""Determine the changes between a git tree and a working tree with index.
"""
to_tree_sha, extras = target.git_snapshot(
want_unversioned=want_unversioned)
store = OverlayObjectStore([source_store, target.store])
return tree_changes(store,
from_tree_sha,
to_tree_sha,
include_trees=include_trees,
rename_detector=rename_detector,
want_unchanged=want_unchanged,
change_type_same=True), extras
def test_hard_head(self):
fullpath = os.path.join(self.repo.path, 'foo')
with open(fullpath, 'w') as f:
f.write("BAR")
porcelain.add(self.repo.path, paths=[fullpath])
porcelain.commit(self.repo.path, message=b"Some message",
committer=b"Jane <*****@*****.**>",
author=b"John <*****@*****.**>")
with open(os.path.join(self.repo.path, 'foo'), 'wb') as f:
f.write(b"OOH")
porcelain.reset(self.repo, "hard", b"HEAD")
index = self.repo.open_index()
changes = list(tree_changes(self.repo,
index.commit(self.repo.object_store),
self.repo[b'HEAD'].tree))
self.assertEqual([], changes)
def test_simple(self):
"""
Basic test of porcelain push where self.repo is the remote. First
clone the remote, commit a file to the clone, then push the changes
back to the remote.
"""
outstream = StringIO()
errstream = StringIO()
porcelain.commit(repo=self.repo.path, message='init',
author='', committer='')
# Setup target repo cloned from temp test repo
clone_path = tempfile.mkdtemp()
porcelain.clone(self.repo.path, target=clone_path, outstream=outstream)
# create a second file to be pushed back to origin
handle, fullpath = tempfile.mkstemp(dir=clone_path)
porcelain.add(repo=clone_path, paths=[os.path.basename(fullpath)])
porcelain.commit(repo=clone_path, message='push',
author='', committer='')
# Setup a non-checked out branch in the remote
refs_path = os.path.join('refs', 'heads', 'foo')
self.repo[refs_path] = self.repo['HEAD']
# Push to the remote
porcelain.push(clone_path, self.repo.path, refs_path, outstream=outstream,
errstream=errstream)
# Check that the target and source
r_clone = Repo(clone_path)
# Get the change in the target repo corresponding to the add
# this will be in the foo branch.
change = list(tree_changes(self.repo, self.repo['HEAD'].tree,
self.repo['refs/heads/foo'].tree))[0]
self.assertEquals(r_clone['HEAD'].id, self.repo[refs_path].id)
self.assertEquals(os.path.basename(fullpath), change.new.path)
def dulwichCommit(self, filePath, fullPath, kind): git = Repo(AUTOGIT_PATH) staged = map(str,[filePath]) git.stage( staged ) index = git.open_index() try: committer = git._get_user_identity() except ValueError: committer = "autogit" try: head = git.head() except KeyError: return git.do_commit( '%s - autogit commit (via dulwich)' % kind, committer=committer) changes = list(tree_changes(git, index.commit(git.object_store), git['HEAD'].tree)) if changes and len(changes) > 0: return git.do_commit( '%s - autogit commit (via dulwich)' % kind, committer=committer) return None
def diff( self, old_sha):
"""
Traverses the diff tree changes and returns a list of changes by change type.
:param old_sha: parent commit's sha-1
:return: map of objects that changed where the key is the change type and the value is a list of
lists of tuples
"""
orig = self._get_object(ROOT_PATH, commit_sha=old_sha)
new = self._get_object(ROOT_PATH)
keys = { diff_tree.CHANGE_DELETE: 'delete',
diff_tree.CHANGE_ADD: 'add',
diff_tree.CHANGE_MODIFY: 'modify'}
out = { k: [] for k in keys.values() }
for change_tree in diff_tree.tree_changes(self.repo.object_store, orig.id, new.id, want_unchanged=False):
if change_tree.type.lower() == "delete" and change_tree.old.path:
# if the change was a delete, we have no tree or blob to yield so return key with no value
# return in the same type of structure for consistency
out[change_tree.type].append([(change_tree.old.path, None)])
else:
out[change_tree.type].append(filter(None, self.entries(change_tree.new.path)))
return out
def show_commit(request, dir_name, commit_id, files_extenshion=None):
"""Return changes make in current commit
data -- include blocks code of each modify files.
"""
commit = get_commit_by_rep_commit_id(dir_name, commit_id)
pth = path.join(settings.REPOS_PATH, dir_name)
repository = repo.Repo(pth)
data = []
# used encode('latin-1') below to solve some problem with unicode
# and bytestring
commit = repository[commit_id.encode('latin-1')]
if len(commit.parents) == 0:
parent = None
else:
parent = repository[commit.parents[0]].tree
delta = diff_tree.tree_changes(repository, parent, commit.tree)
for item in delta:
block = []
old = ""
if item.old.sha:
old = repository[item.old.sha].data.split("\n")
new = repository[item.new.sha].data.split("\n")
for line in unified_diff(old, new):
block.append(line)
data.append(
(item.old.path, item.new.path, block)
)
context = {'data': data}
context = prepare_context(context, request.user)
return render(request, "octonyan/commit_info.html", context)
def _determine_relative_changes_from_commits(self, from_ref, to_ref): # noqa: C901
if (not from_ref) or (not to_ref):
return
try:
_r = Repo(self._project_root)
except NotGitRepository as ngr:
raise TaskCatException(
f"Directory ({self._project_root}) is not a git repository"
) from ngr
for change in tree_changes(_r.object_store, _r[from_ref].tree, _r[to_ref].tree):
if change.type in ["add", "modify"]:
_c = Path(self._project_root / change.new.path.decode()).resolve()
if self._single_package_path:
if _c.parent != self._single_package_path:
continue
self._dirs_with_changes.add(_c.parent)
if change.type in ["delete"]:
_c = Path(self._project_root / change.old.path.decode()).resolve()
if self._single_package_path:
if _c.parent != self._single_package_path:
continue
self._dirs_with_changes.add(_c.parent)
def assertChangesEqual(self, expected, tree1, tree2, **kwargs):
actual = list(tree_changes(self.store, tree1.id, tree2.id, **kwargs))
self.assertEqual(expected, actual)
def get_branch_changed_entries(self, branch_name):
"""Return the name of all changed files within the branch."""
master_tree = self._get_tree(self.get_branch('master'))
branch_tree = self._get_tree(self.get_branch(branch_name))
return tree_changes(self._repo.object_store, branch_tree.id,
master_tree.id)
def getAllLogsMy(self, file = None, r1 = None, r2 = None ,reverse=True, follow = False, dolower=False, toText =False):
"""
r2 inclusive
r1 exclusive
(r1,r2]
"""
if file is not None and isinstance(file, str):
file = file.encode()
if r1 is not None and isinstance(r1, str):
r1 = r1.encode()
if r2 is not None and isinstance(r2, str):
r2 = r2.encode()
out = []
p = file
walker = None
prev = None
if file==None:
if r2 is not None:
walker = self.r.get_graph_walker(heads = [r2])
else:
walker = self.r.get_graph_walker()
else:
if r2 is not None:
walker = iter(self.r.get_walker(paths=[file],follow = follow, include = [r2]))
else:
walker = iter(self.r.get_walker(paths=[file],follow = follow))
if file is not None:
cset = next(walker, None)
else:
cset = walker.next()
while True:
commit = None
if cset!=None:
if file is not None:
commit = cset.commit
else:
commit = self.r.get_object(cset)
if prev is None:
prev = commit
if file is not None:
cset = next(walker, None)
else:
cset = walker.next()
continue
if file is not None:
files = [file]
else:
files = []
tr = None
if commit is not None:
tr = commit.tree
delta = diff_tree.tree_changes(self.r, tr, prev.tree)
for x in delta:
if x.new.path is not None:
files.append(x.new.path)
else:
if x.old.path is not None:
files.append(x.old.path)
msg = prev.message
if toText:
msg = Helpers.ConvertToUTF8(msg)
if dolower:
msg = msg.lower()
cid = Helpers.ConvertToUTF8(prev.id)
out.append({'m':msg, 'committer':Helpers.ConvertToUTF8(prev.committer), 'author': Helpers.ConvertToUTF8( prev.author)
, 'commit_time': prev.commit_time, 'commit_timezone': prev.commit_timezone, 'author_time': prev.author_time
, 'author_timezone': prev.author_timezone, 'parents':[Helpers.ConvertToUTF8(pr) for pr in prev.parents], 'cid':cid, 'locid': cid, 'cidshort':cid,
'tree': Helpers.ConvertToUTF8(prev.tree), 'files':[Helpers.ConvertToUTF8(f) for f in files], 'date' : prev.commit_time,'timestamp' : prev.commit_time, 'tags':'', 'branches':''})
prev = commit
if prev == None:
break
if (r1 is not None) and (commit.id == r1):
break
if file is not None:
cset = next(walker, None)
else:
cset = walker.next()
if reverse:
out.reverse()
return out
def getLogLocIds (self,file = None,r1 = None,r2 = None,reverse=True, follow = False, returnLogs = False, ignoreMerges = False):
"""
r2 inclusive
r1 exclusive
(r1,r2]
"""
out = []
auth = []
p = file
walker = None
if file is not None and isinstance(file, str):
file = file.encode()
if r1 is not None and isinstance(r1, str):
r1 = r1.encode()
if r2 is not None and isinstance(r2, str):
r2 = r2.encode()
if file==None:
if r2 is not None:
walker = self.r.get_graph_walker(heads = [r2])
else:
walker = self.r.get_graph_walker()
else:
if r2 is not None:
walker = iter(self.r.get_walker(paths=[file],follow = follow, include = [r2]))
else:
walker = iter(self.r.get_walker(paths=[file],follow = follow))
prev = None
if file is not None:
cset = next(walker, None)
else:
cset = walker.next()
while True:
commit = None
if cset!=None:
if file is not None:
commit = cset.commit
else:
commit = self.r.get_object(cset)
if prev is None:
prev = commit
if file is not None:
cset = next(walker, None)
else:
cset = walker.next()
continue
tr = None
if commit is not None:
tr = commit.tree
delta = diff_tree.tree_changes(self.r, tr, prev.tree)
for x in delta:
#if isinstance(x, Random):
# continue
if file is not None:
if x.new.path == file or x.old.path == file:
if returnLogs:
out.append(x)
auth.append(prev.author)
else:
if prev.id not in out:
out.append(prev.id)
auth.append(prev.author)
else:
if returnLogs:
out.append(x)
auth.append(prev.author)
else:
if prev.id not in out:
out.append(prev.id)
auth.append(prev.author)
prev = commit
if prev == None:
break
if (r1 is not None) and (prev.id == r1):
break
if file is not None:
cset = next(walker, None)
else:
cset = walker.next()
if reverse:
if not returnLogs:
out = [Helpers.ConvertToUTF8(o) for o in out]
out.reverse()
auth.reverse()
return out, [Helpers.ConvertToUTF8(a) for a in auth]
def getChangedFiles(self,r1,r2):
if file is not None and isinstance(file, str):
file = file.encode()
if r1 is not None and isinstance(r1, str):
r1 = r1.encode()
if r2 is not None and isinstance(r2, str):
r2 = r2.encode()
out = set()
outmod = set()
outdel = set()
types = set()
if r2 is not None:
walker = self.r.get_graph_walker(heads=[r2])
else:
walker = self.r.get_graph_walker()
cset = walker.next()
prev = None
if cset == None:
return None
while True:
commit = None
if cset!=None:
commit = self.r.get_object(cset)
if prev is None:
prev = commit
cset = walker.next()
continue
tr = None
if commit is not None:
tr = commit.tree
delta = diff_tree.tree_changes(self.r, tr, prev.tree)
for x in delta:
if x.type == 'delete':
outdel.add(x.old.path)
elif x.type == 'add':
out.add(x.new.path)
else:
outmod.add(x.new.path)
types.add(x.type)
#try:
# out.add(x.new.path)
#except Exception as ex:
# pass
if commit is None:
break
prev = commit
if (r1 is not None) and (prev.id == r1):
break
cset = walker.next()
if None in out:
out.remove(None)
return sorted(list(out)), sorted(list(outdel)), sorted(list(outmod)), sorted(list(types))
def assertChangesEqual(self, expected, tree1, tree2, **kwargs):
actual = list(tree_changes(self.store, tree1.id, tree2.id, **kwargs))
self.assertEqual(expected, actual)
def _merge_branches(self, base, mine, other, take_mine=False):
def load_json(path, branch):
try:
blob = self.blob(path, branch)
except (KeyError, TypeError):
return {}
else:
return loads(blob.as_raw_string())
merge_tree = Tree()
base_tree, my_tree, other_tree = (self._get_tree(x)
for x in (base, mine, other))
ren_detector = RenameDetector(self._repo.object_store)
conflicts = {}
my_changes, other_changes = (tree_changes(self._repo.object_store,
base_tree.id,
x.id,
want_unchanged=True,
rename_detector=ren_detector)
for x in (my_tree, other_tree))
changes_by_path = defaultdict(list)
for change in chain(my_changes, other_changes):
if change.type == CHANGE_DELETE or change.type == CHANGE_RENAME:
path = change.old.path
else:
path = change.new.path
changes_by_path[path].append(change)
had_conflict = False
for path, changes in changes_by_path.items():
if len(changes) == 2:
my_changes, other_changes = changes
if my_changes.type == CHANGE_DELETE:
if other_changes.type in (CHANGE_RENAME, CHANGE_MODIFY):
merge_tree.add(other_changes.new.path, FILE_MODE,
other_changes.new.sha)
else:
continue
elif other_changes.type == CHANGE_DELETE:
if my_changes.type in (CHANGE_RENAME, CHANGE_MODIFY):
merge_tree.add(my_changes.new.path, FILE_MODE,
my_changes.new.sha)
else:
continue
else:
jsons = [load_json(path, x) for x in (base, mine, other)]
base_json, my_json, other_json = jsons
# When dealing with renames, file contents are under the
# 'new' path. Note that the file will be finally stored
# under the name given by the last rename.
if other_changes.type == CHANGE_RENAME:
other_json = load_json(other_changes.new.path, other)
path = other_changes.new.path
if my_changes.type == CHANGE_RENAME:
my_json = load_json(my_changes.new.path, mine)
path = my_changes.new.path
if take_mine:
merged_json = my_json or other_json or base_json
else:
merged_json, merge_conflict = merge_jsons(*jsons)
if merge_conflict:
conflicts[path] = merged_json
had_conflict = had_conflict or merge_conflict
merged_blob = Blob.from_string(
dumps(merged_json, sort_keys=True, indent=4))
self._update_store(merged_blob)
merge_tree.add(path, FILE_MODE, merged_blob.id)
else:
data = (load_json(path, mine) or load_json(path, other)
or load_json(path, base))
blob = Blob.from_string(dumps(data, sort_keys=True, indent=4))
self._update_store(blob)
merge_tree.add(path, FILE_MODE, blob.id)
self._update_store(merge_tree)
return merge_tree, conflicts
def get_branch_changed_entries(self, branch_name):
"""Return the name of all changed files within the branch."""
master_tree = self._get_tree(self.get_branch('master'))
branch_tree = self._get_tree(self.get_branch(branch_name))
return tree_changes(
self._repo.object_store, branch_tree.id, master_tree.id)
def _merge_branches(self, base, mine, other, take_mine=False):
def load_raw(path, branch):
try:
blob = self.blob(path, branch)
except (KeyError, TypeError):
return '{}'
else:
return blob.as_raw_string()
def load_json(path, branch):
return loads(load_raw(path, branch))
merge_tree = Tree()
base_tree, my_tree, other_tree = (self._get_tree(x)
for x in (base, mine, other))
ren_detector = RenameDetector(self._repo.object_store)
conflicts = {}
my_changes, other_changes = (
tree_changes(
self._repo.object_store,
base_tree.id,
x.id,
want_unchanged=True,
rename_detector=ren_detector)
for x in (my_tree, other_tree))
changes_by_path = defaultdict(list)
for change in chain(my_changes, other_changes):
if change.type == CHANGE_DELETE or change.type == CHANGE_RENAME:
path = change.old.path
else:
path = change.new.path
changes_by_path[path].append(change)
had_conflict = False
for path, changes in changes_by_path.items():
if len(changes) == 2:
my_changes, other_changes = changes
if my_changes.type == CHANGE_DELETE:
if other_changes.type in (CHANGE_RENAME, CHANGE_MODIFY):
merge_tree.add(other_changes.new.path,
FILE_MODE, other_changes.new.sha)
else:
continue
elif other_changes.type == CHANGE_DELETE:
if my_changes.type in (CHANGE_RENAME, CHANGE_MODIFY):
merge_tree.add(my_changes.new.path,
FILE_MODE, my_changes.new.sha)
else:
continue
else:
try:
jsons = [load_json(path, x)
for x in (base, mine, other)]
except ValueError: # Handle non json data
blob = Blob.from_string(load_raw(path, mine))
self._update_store(blob)
merge_tree.add(path, FILE_MODE, blob.id)
continue
base_json, my_json, other_json = jsons
# When dealing with renames, file contents are under the
# 'new' path. Note that the file will be finally stored
# under the name given by the last rename.
if other_changes.type == CHANGE_RENAME:
other_json = load_json(other_changes.new.path, other)
path = other_changes.new.path
if my_changes.type == CHANGE_RENAME:
my_json = load_json(my_changes.new.path, mine)
path = my_changes.new.path
if take_mine:
merged_json = my_json or other_json or base_json
else:
merged_json, merge_conflict = merge_jsons(*jsons)
if merge_conflict:
conflicts[path] = merged_json
had_conflict = had_conflict or merge_conflict
merged_blob = Blob.from_string(
dumps(merged_json, sort_keys=True, indent=4))
self._update_store(merged_blob)
merge_tree.add(path, FILE_MODE, merged_blob.id)
else:
try:
data = (load_json(path, mine) or load_json(path, other) or
load_json(path, base))
except ValueError: # Loading a non json file
blob = Blob.from_string(load_raw(path, mine))
else:
blob = Blob.from_string(dumps(data, sort_keys=True,
indent=4))
self._update_store(blob)
merge_tree.add(path, FILE_MODE, blob.id)
self._update_store(merge_tree)
return merge_tree, conflicts
def _merge_branches(self, base, mine, other):
def load_json(path, branch):
try:
blob = self.blob(path, branch)
except KeyError:
return {}
else:
return loads(blob.as_raw_string())
merge_tree = Tree()
base_tree, my_tree, other_tree = (self._get_tree(x)
for x in (base, mine, other))
ren_detector = RenameDetector(self._repo.object_store)
my_changes, other_changes = (
tree_changes(
self._repo.object_store,
base_tree.id,
x.id,
want_unchanged=True,
rename_detector=ren_detector)
for x in (my_tree, other_tree))
changes_by_path = defaultdict(list)
for change in chain(my_changes, other_changes):
if change.type == CHANGE_DELETE or change.type == CHANGE_RENAME:
path = change.old.path
else:
path = change.new.path
changes_by_path[path].append(change)
had_conflict = False
for path, changes in changes_by_path.iteritems():
if len(changes) == 2:
my_changes, other_changes = changes
if my_changes.type == CHANGE_DELETE:
if other_changes.type in (CHANGE_RENAME, CHANGE_MODIFY):
merge_tree.add(other_changes.new.path,
FILE_MODE, other_changes.new.sha)
else:
continue
elif other_changes.type == CHANGE_DELETE:
if my_changes.type in (CHANGE_RENAME, CHANGE_MODIFY):
merge_tree.add(my_changes.new.path,
FILE_MODE, my_changes.new.sha)
else:
continue
else:
jsons = [load_json(path, x) for x in (base, mine, other)]
# When dealing with renames, file contents are under the
# 'new' path. Note that the file will be finally stored
# under the name given by the last rename.
if other_changes.type == CHANGE_RENAME:
jsons[2] = load_json(other_changes.new.path, other)
path = other_changes.new.path
if my_changes.type == CHANGE_RENAME:
jsons[1] = load_json(my_changes.new.path, mine)
path = my_changes.new.path
merged_json, merge_conflict = merge_jsons(*jsons)
had_conflict = had_conflict or merge_conflict
merged_blob = Blob.from_string(
dumps(merged_json, sort_keys=True, indent=4))
self._update_store(merged_blob)
merge_tree.add(path, FILE_MODE, merged_blob.id)
else:
merge_tree.add(path, FILE_MODE, changes[0].new.sha)
self._update_store(merge_tree)
return merge_tree, had_conflict
def main():
setup()
while True:
#Try to update the repo
updateRepo(SOURCE_FOLDER)
HEAD = Source.head()
FETCH_HEAD = Source.ref("FETCH_HEAD")
#Check if there are any updates
if HEAD == FETCH_HEAD:
time.sleep(60)
continue
#Build the tree of commits to build
commits = []
listCommits(Source, FETCH_HEAD, HEAD, commits)
commits.reverse()
base_tree = Source.commit(HEAD).tree
for c in commits:
global CURRENT_COMMIT
CURRENT_COMMIT = c.id
switchTo(SOURCE_FOLDER, c.id)
print("Executing commit " + c.id)
#Iterate through the difference tree
diff = tree_changes(Source, base_tree, c.tree)
steps = [False for x in BUILD_STEPS]
for d in diff:
#Handle deletes
if d.type == 'delete':
fname = d.old.path
if publish_file(fname):
os.remove(os.path.join(DEST_FOLDER, fname))
Dest.stage(fname)
continue
#Copy over all the files
fname = d.new.path
if not fname:
continue
if publish_file(fname):
copy(fname)
Dest.stage(fname)
for i, st in enumerate(BUILD_STEPS):
if st.neededBy(fname):
steps[i] = True
#Execute all build steps
cursteps = []
for i, needed in enumerate(steps):
if not needed:
continue
step = BUILD_STEPS[i]()
step.execute()
while not step.finished():
time.sleep(1)
cursteps.append(step)
#Finalize all build steps
for step in cursteps:
step.finalize()
#Set up the commit in the new repo
Dest.do_commit(
message=c.message +
"\n\nOriginal Commit: BlindMindStudios/StarRuler2@" + c.id,
committer=c.committer,
author=c.author,
commit_timestamp=c._commit_time,
commit_timezone=c._commit_timezone,
author_timestamp=c._author_time,
author_timezone=c._author_timezone)
#Set up for next
base_tree = c.tree
#Push this individual commit
pushRepo(DEST_FOLDER)
