def write_file(repo: pygit2.Repository, tree: pygit2.Tree, filepath: str,
contents: str) -> pygit2.Oid:
blob = repo.create_blob(contents.encode("utf-8"))
paths = filepath.split("/")
trees = [tree]
for path in paths[:-1]:
try:
to_insert = repo[trees[0][path].oid]
except KeyError:
to_insert = None
trees.insert(0, to_insert)
to_insert = blob
for path in reversed(paths):
tree = trees.pop(0)
assert isinstance(to_insert, pygit2.Oid)
assert isinstance(repo[to_insert], pygit2.Blob) or isinstance(
repo[to_insert], pygit2.Tree)
if tree is None:
tb = repo.TreeBuilder()
else:
tb = repo.TreeBuilder(tree)
tb.insert(
path, to_insert, pygit2.GIT_FILEMODE_BLOB if isinstance(
repo[to_insert], pygit2.Blob) else pygit2.GIT_FILEMODE_TREE)
to_insert = tb.write()
assert len(trees) == 0
return to_insert
def update_tree(
repo: git.Repository, tree: git.Oid, path: List[str], content: str
) -> git.Oid:
"""
adds a blob with `content` at `path` to `tree` in `repo`
>>> repo = create_repo()
>>> tree = repo.TreeBuilder().write()
>>> for i in range(10):
... path = store_hash(f"{i}")
... content = nar_hash(path)
... tree = update_tree(repo, tree, common.shards(path, depth=5), content)
>>> print(tree)
00f68bdb866b654d4ce3da90609b74137605bd90
"""
for entry in repo.get(tree):
# subdir exists: recurse
if (entry.name == path[0]) and (entry.type == "tree"):
sub = update_tree(repo, entry.id, path[1:], content)
builder = repo.TreeBuilder(repo.get(tree))
builder.remove(path[0])
builder.insert(path[0], sub, git.GIT_FILEMODE_TREE)
return builder.write()
# subdir does not exist: create required objects
if len(path) > 1:
# write leaf node
sub = update_tree(repo, repo.TreeBuilder().write(), [path[-1]], content)
# build intermediate nodes
for d in reversed(path[1:-1]):
builder = repo.TreeBuilder()
builder.insert(d, sub, git.GIT_FILEMODE_TREE)
sub = builder.write()
# attach to `tree`
builder = repo.TreeBuilder(repo.get(tree))
builder.insert(path[0], sub, git.GIT_FILEMODE_TREE)
return builder.write()
# path[0] is not a subdir: write blob
elif len(path) == 1:
blob = repo.write(git.GIT_OBJ_BLOB, content)
builder = repo.TreeBuilder(repo.get(tree))
builder.insert(path[0], blob, git.GIT_FILEMODE_BLOB)
return builder.write()
else:
raise Exception(f"invalid path: {path}")
class ChangeCtxBaseTestCase(unittest.TestCase):
def setUp(self):
self.repo_path = mkdtemp()
init_repository(self.repo_path, False)
self.repo = Repository(self.repo_path)
# create files and commit
self.sign = Signature('foo', '*****@*****.**')
self.repo_files = ['a%i.rst' % i for i in range(5)]
for i in self.repo_files:
with codecs.open(os.path.join(self.repo_path, i),
'w',
encoding='utf-8') as fp:
fp.write('dumb file %s\n' % i)
self.tree = self.repo.TreeBuilder()
self.old_commit = git_commit(self.repo, self.tree, self.repo_files)
def tearDown(self):
try:
rmtree(self.repo_path)
except:
pass
@property
def ctx_class(self):
raise NotImplementedError
def get_ctx(self):
return self.ctx_class(self.repo_path)
class FileCtxTestCase(unittest.TestCase):
def setUp(self):
self.repo_path = mkdtemp()
init_repository(self.repo_path, False)
self.repo = Repository(self.repo_path)
self.file_name = 'foo.rst'
self.file_path = os.path.join(self.repo_path, self.file_name)
with codecs.open(self.file_path, 'w', encoding='utf-8') as fp:
fp.write('test\n')
self.tree = self.repo.TreeBuilder()
self.last_commit = git_commit(self.repo, self.tree, [self.file_name])
self.changectx = self.repo.head
def tearDown(self):
try:
rmtree(self.repo_path)
except:
pass
def test_path(self):
ctx = FileCtx(self.repo, self.changectx, self.file_name)
self.assertEqual(ctx.path, 'foo.rst')
def test_content(self):
ctx = FileCtx(self.repo, self.changectx, self.file_name)
self.assertEqual(ctx.content, 'test\n')
with codecs.open(self.file_path, 'a', encoding='utf-8') as fp:
fp.write('lol\n') # change file without git add
ctx = FileCtx(self.repo, self.changectx, self.file_name)
self.assertEqual(ctx.content, 'test\n')
def test_content_from_index_read_without_git_add(self):
ctx = FileCtx(self.repo, self.changectx, self.file_name, True)
self.assertEqual(ctx.content, 'test\n')
with codecs.open(self.file_path, 'a', encoding='utf-8') as fp:
fp.write('lol\n') # change file without git add
ctx = FileCtx(self.repo, self.changectx, self.file_name, True)
self.assertEqual(ctx.content, 'test\nlol\n')
def test_author(self):
ctx = FileCtx(self.repo, self.changectx, self.file_name)
self.assertEqual(ctx.author, 'foo <*****@*****.**>')
def test_date_and_mdate(self):
ctx = FileCtx(self.repo, self.changectx, self.file_name)
time.sleep(1)
self.assertTrue(ctx.date < time.time())
self.assertTrue(ctx.mdate is None)
old_date = ctx.date
time.sleep(1)
with codecs.open(self.file_path, 'a', encoding='utf-8') as fp:
fp.write('foo\n')
git_commit(self.repo, self.tree, [self.file_name], [self.last_commit])
ctx = FileCtx(self.repo, self.changectx, self.file_name)
self.assertEqual(ctx.date, old_date)
self.assertTrue(ctx.mdate > old_date)
def get_empty_tree_hash(repo: pygit2.Repository) -> str:
"""
The hash for the empty tree can be generated using: `git hash-object -t tree /dev/null`
(see this Stack Overflow post for more details: https://stackoverflow.com/q/9765453)
We prefer to generate the empty hash using the pygit2 Repository's TreeBuilder, which avoids use
of the SHA-1 magic number: `4b825dc642cb6eb9a060e54bf8d69288fbee4904`.
This makes locust as future-compatible as pygit2.
"""
tree_builder = repo.TreeBuilder()
return str(tree_builder.write())
class GitRepositoryTestCase(unittest.TestCase):
def setUp(self):
self.repo_path = mkdtemp()
init_repository(self.repo_path, False)
self.repo = Repository(self.repo_path)
def tearDown(self):
try:
rmtree(self.repo_path)
except:
pass
def test_create_repo(self):
repo_path = mkdtemp()
try:
GitRepository.create_repo(repo_path)
for f in [
os.path.join('content', 'attachments', 'mercurial.png'),
os.path.join('content', 'post', 'example-post.rst'),
os.path.join('content', 'post', 'lorem-ipsum.rst'),
os.path.join('content', 'about.rst'),
os.path.join('static', 'screen.css'),
os.path.join('templates', 'base.html'),
os.path.join('templates', 'posts.html'),
os.path.join('templates', 'post_list.html'), 'config.yaml',
'.gitignore', '.git'
]:
self.assertTrue(os.path.exists(os.path.join(repo_path, f)),
'Not found: %s' % f)
finally:
rmtree(repo_path)
def test_get_changectx_rev_default(self):
git_repo = GitRepository(self.repo_path)
with codecs.open(os.path.join(self.repo_path, 'foo.rst'),
'w',
encoding='utf-8') as fp:
fp.write('foo')
sign = Signature('foo', '*****@*****.**')
tree = self.repo.TreeBuilder().write()
self.repo.index.add('foo.rst')
self.repo.create_commit('refs/heads/master', sign, sign, 'foo', tree,
[])
self.assertTrue(
isinstance(git_repo.get_changectx(REVISION_DEFAULT),
ChangeCtxDefault),
'changectx object is not an instance of '
'ChangeCtxDefault')
def test_get_changectx_rev_working_dir(self):
git_repo = GitRepository(self.repo_path)
with codecs.open(os.path.join(self.repo_path, 'foo.rst'),
'w',
encoding='utf-8') as fp:
fp.write('foo')
sign = Signature('foo', '*****@*****.**')
tree = self.repo.TreeBuilder().write()
self.repo.index.add('foo.rst')
self.repo.create_commit('refs/heads/master', sign, sign, 'foo', tree,
[])
self.assertTrue(
isinstance(git_repo.get_changectx(REVISION_WORKING_DIR),
ChangeCtxWorkingDir),
'changectx object is not an instance of ChangeCtxWorkingDir')
class PyGitEngine(GitContentDatabaseEngine):
def __init__(self, config):
super(PyGitEngine, self).__init__(config)
self.repo = None
def connect(self):
"""Create content directory"""
if not isdir(self.content_path):
init_repository(self.content_path, bare=True)
self.repo = Repository(self.content_path)
self.create_initial_commit()
else:
self.repo = Repository(self.content_path)
@staticmethod
def do_put(content_path, object_hashes, content, filename):
"""Perform put operation. This is used in the distributed wrapper"""
content_hash = Repository(content_path).create_blob(content)
result = object_hashes[filename] = str(content_hash)
return result
def put_attr(self, content, filename):
"""Return attributes for the do_put operation"""
filename = self._inc_name(filename)
return (self.content_path, self.object_hashes, content, filename)
def put(self, content, filename="generic"): # pylint: disable=method-hidden
"""Put content in the content database"""
return self.do_put(*self.put_attr(content, filename))
def get(self, content_hash): # pylint: disable=method-hidden
"""Get content from the content database"""
return_data = self.repo[content_hash].data
return return_data
def find_subhash(self, content_hash):
"""Find hash in git"""
try:
blob = self.repo.revparse_single(content_hash)
return str(blob.id)
except KeyError:
return None
def create_initial_commit(self):
"""Create the initial commit of the git repository"""
empty_tree = self.repo.TreeBuilder().write()
self.create_commit_object(self._initial_message, empty_tree)
def create_commit_object(self, message, tree):
"""Create a commit object"""
references = list(self.repo.references)
master_ref = self.repo.lookup_reference(
self._commit_ref) if len(references) > 0 else None
parents = []
if master_ref is not None:
parents = [master_ref.peel().id]
author = Signature(self._commit_name, self._commit_email)
return self.repo.create_commit(self._commit_ref, author, author,
message, tree, parents)
def new_tree(self, parent):
"""Create new git tree"""
return self.repo.TreeBuilder()
def insert_blob(self, tree, basename, value):
"""Insert blob into tree"""
tree.insert(basename, value, GIT_FILEMODE_BLOB)
def insert_tree(self, tree, basename, value):
"""Insert tree into tree"""
tree.insert(basename, value, GIT_FILEMODE_TREE)
def write_tree(self, tree):
"""Write tree to git directory"""
return tree.write()
class vgit_repo(metaclass=singleton):
def __init__(self):
self.repo = None
def vgit_load_repo(self, path):
if path is None:
self.repo = None
return
try:
self.repo = Repository(path)
except ValueError:
self.repo = None
except Exception:
self.repo = None
def vgit_clone(self, path, url, log_cb):
"""Wraper method to clone a
repository from a given `url' to a given `path'"""
try:
pygit2.clone_repository(url, path)
except ValueError as err:
log_cb(str(err))
return False
except Exception as err:
log_cb(str(err))
return False
return True
def vgit_init(self, path, log_cb, bare=False):
"""Wraper method to init a repository in given `path'"""
try:
pygit2.init_repository(path, bare)
except ValueError as err:
log_cb(str(err))
return False
except Exception as err:
log_cb(str(err))
return False
return True
def vgit_add_all(self, log_cb):
"""Wraper method to add all files to a commit."""
try:
if self.repo is None:
log_cb(
"No .git in current directory. Use `init' to crete a new repository."
)
return False
self.repo.index.add_all()
self.repo.index.write()
except ValueError as err:
log_cb(str(err))
return False
except Exception as err:
log_cb(str(err))
return False
return True
def vgit_commit(self, user_name, email, branch, message, log_cb):
"""Wraper method for the commit -m "message" git command. `user_name' and
`email' informations of the autor of the commit
`user_name_author' and email_author: is the same idea of the
commiter but for the author `branch' branch which the user
want's to do the commit `message': message of the commit"""
try:
if self.repo is None:
log_cb(
"No .git in current directory. Use `init' to crete a new repository."
)
return False
reference = 'refs/heads/' + branch
author = pygit2.Signature(user_name, email)
# usando author e commiter como o mesmo ser
self.repo.create_commit(reference, author, author, message,
self.repo.TreeBuilder().write(),
[self.repo.head.target])
except ValueError as err:
log_cb(str(err))
return False
except Exception as err:
log_cb(str(err))
return False
return True
def vgit_commits(self, log_cb):
if self.repo is None:
log_cb("No commits...")
else:
for commit in self.repo.walk(self.repo[self.repo.head.target].id,
pygit2.GIT_SORT_TIME):
log_cb('\n'.join([
'Commit: #{}'.format(commit.tree_id.hex),
'Author: {} <{}>'.format(commit.author.name,
commit.author.email), 'Message: ',
commit.message, ''
]))
def vgit_push(self, path, message, user_name_commiter, user_name_author,
email_commiter, email_author, branch, user_name_pusher,
user_passoword_pusher, log_cb):
try:
repo = Repository(path)
index = repo.index
reference = 'refs/heads/' + branch
tree = index.write_tree()
author = pygit2.Signature(user_name_author, email_author)
commiter = pygit2.Signature(user_name_commiter, email_commiter)
oid = repo.create_commit(reference, author, commiter, message,
tree, [repo.head.target])
credentials = pygit2.UserPass(user_name_pusher,
user_passoword_pusher)
remo = repo.remotes["origin"]
remo.credentials = credentials
aux = remo.url
repo.remotes.set_push_url(user_name_pusher, aux)
callbacks = pygit2.RemoteCallbacks(credentials=credentials)
remo.push([reference], callbacks=callbacks)
except ValueError as err:
log_cb(str(err))
return False
except Exception as err:
log_cb(str(err))
return False
return True
class GitHandler(object):
def __init__(self, path, repo_path=None, update_working_copy=True):
"""
Start a git handler in given repository.
`update_working_copy`: wether also to update the working copy.
By default, the git handler will only work on the git database.
Updating the working copy can take a lot of time in
large repositories.
"""
self.path = path
if repo_path is None:
repo_path = self.path
self.repo_path = repo_path
self.update_working_copy = update_working_copy
self.repo = Repository(self.repo_path)
self.working_tree = self.get_last_tree()
self.tree_modifier = TreeModifier(self.repo, self.working_tree)
self.messages = []
print("Started libgit2 git handler in ", self.path)
def get_last_tree(self):
if self.repo.head_is_unborn:
tree_id = self.repo.TreeBuilder().write()
return self.repo[tree_id]
commit = self.repo[self.getCurrentCommit()]
return commit.tree
def insert_into_working_tree(self, blob_id, filename):
self.tree_modifier.insert_blob(blob_id, filename)
def remove_from_working_tree(self, filename):
self.tree_modifier.remove_blob(filename)
def write_file(self, filename, content):
# TODO: combine writing many files
assert isinstance(content, text_type)
data = content.encode('utf-8')
existing_entry = get_tree_entry(self.repo, self.working_tree, filename)
if existing_entry:
type = 'M'
if existing_entry.id == git_hash(data):
return
else:
type = 'A'
blob_id = self.repo.create_blob(data)
self.insert_into_working_tree(blob_id, filename)
if not self.repo.is_bare and self.update_working_copy:
real_filename = os.path.join(self.path, filename)
mkdir_p(os.path.dirname(real_filename))
with codecs.open(real_filename, 'w', encoding='utf-8') as outfile:
outfile.write(content)
self.messages.append(' {} {}'.format(type, filename))
def remove_file(self, filename):
existing_entry = get_tree_entry(self.repo, self.working_tree, filename)
if existing_entry:
self.remove_from_working_tree(filename)
if not self.repo.is_bare and self.update_working_copy:
remove_file_with_empty_parents(self.path, filename)
self.messages.append(' D {}'.format(filename))
def move_file(self, old_filename, new_filename):
self.tree_modifier.move(old_filename, new_filename)
if not self.repo.is_bare and self.update_working_copy:
real_old_filename = os.path.join(self.path, old_filename)
real_new_filename = os.path.join(self.path, new_filename)
mkdir_p(os.path.dirname(real_new_filename))
os.rename(real_old_filename, real_new_filename)
remove_file_with_empty_parents(self.path, old_filename)
self.messages.append(' R {} -> {}'.format(old_filename,
new_filename))
def commit(self):
if self.tree_modifier.tree.oid != self.get_last_tree().oid:
raise Exception("The repository was modified outside of this process. For safety reasons, we cannot commit!")
self.working_tree = self.tree_modifier.apply()
self.tree_modifier = TreeModifier(self.repo, self.working_tree)
if self.repo.head_is_unborn:
parents = []
else:
commit = self.repo[self.getCurrentCommit()]
if commit.tree.id == self.working_tree.id:
return
parents = [commit.id]
config = self.repo.config
author = Signature(config['user.name'], config['user.email'])
committer = Signature(config['user.name'], config['user.email'])
tree_id = self.working_tree.id
message = '\n'.join(self.messages)
self.repo.create_commit('refs/heads/master',
author, committer, message,
tree_id,
parents)
self.saveCurrentCommit()
self.messages = []
if not self.repo.is_bare and self.update_working_copy:
self.repo.index.read_tree(self.working_tree)
self.repo.index.write()
def reset(self):
self.working_tree = self.get_last_tree()
self.tree_modifier = TreeModifier(self.repo, self.working_tree)
self.messages = []
def getCurrentCommit(self):
return self.repo.head.target
def saveCurrentCommit(self):
with open(os.path.join(self.path, 'dbcommit'), 'w') as dbcommit_file:
dbcommit_file.write(self.getCurrentCommit().hex+'\n')
