def init_repo(repo_path, clone_from=None, clone_refs=False,
alternate_repo_paths=None, is_bare=True):
"""Initialise a new git repository or clone from existing."""
assert is_valid_new_path(repo_path)
init_repository(repo_path, is_bare)
if clone_from:
# The clone_from's objects and refs are in fact cloned into a
# subordinate tree that's then set as an alternate for the real
# repo. This lets git-receive-pack expose available commits as
# extra haves without polluting refs in the real repo.
sub_path = os.path.join(repo_path, 'turnip-subordinate')
clone_repository(clone_from, sub_path, True)
assert is_bare
alt_path = os.path.join(repo_path, 'objects/info/alternates')
with open(alt_path, 'w') as f:
f.write('../turnip-subordinate/objects\n')
if clone_refs:
# With the objects all accessible via the subordinate, we
# can just copy all refs from the origin. Unlike
# pygit2.clone_repository, this won't set up a remote.
# TODO: Filter out internal (eg. MP) refs.
from_repo = Repository(clone_from)
to_repo = Repository(repo_path)
for ref in from_repo.listall_references():
to_repo.create_reference(
ref, from_repo.lookup_reference(ref).target)
if alternate_repo_paths:
write_alternates(repo_path, alternate_repo_paths)
ensure_config(repo_path) # set repository configuration defaults
return repo_path
def sync_handler(fork_from: str, from_sha: str, repo_name: str,
ticket_id: int, pr_url: str):
output_path = '{}.txt'.format(pr_url.split('/', 3)[3].rsplit('/', 2)[0])
output_path = os.path.join(WORK_DIR, output_path.replace('/', '_'))
work_tree = os.path.join(WORK_DIR, fork_from)
parent_path = os.path.dirname(work_tree)
if not os.path.exists(parent_path):
os.makedirs(parent_path)
if not os.path.exists(work_tree):
repo = clone_repository(
'{0}{1}.git'.format(GITHUB_URL, fork_from), work_tree)
else:
repo = Repository(work_tree)
remote_name = repo_name.split('/')[0]
update_remote(work_tree, repo, repo_name, remote_name)
if remote_name == 'origin':
commit = repo.revparse_single(from_sha)
repo.checkout_tree(commit, strategy=GIT_CHECKOUT_FORCE)
else:
ref_name = 'refs/pull/{0}/head'.format(ticket_id)
try:
repo.create_reference(ref_name, from_sha)
except ValueError:
pass
ref = repo.lookup_reference(ref_name)
repo.checkout(ref, strategy=GIT_CHECKOUT_FORCE)
cwd = os.getcwd()
os.chdir(work_tree)
subprocess.call(
'{} . --output-file={}'.format(FLAKE8_EXECUTABLE, output_path),
shell=True)
os.chdir(cwd)
return output_path
class Git(object):
r"""
Interact with a git repository.
"""
def __init__(self, gitdir):
r"""
Take a path to the git repository. Other methods interact with
this git repository.
"""
self.repo = Repository(gitdir)
def branches(self):
r"""
Return the list of a branch name and its last commit id.
"""
return self._refs('heads')
def tags(self):
r"""
Return the list of a tag name and its last commit id.
"""
return self._refs('tags')
def _refs(self, type):
refs = {}
pattern = re.compile(r'refs/%s/(.*)$' % type)
for ref in self.repo.listall_references():
m = pattern.match(ref)
if m:
reference = self.repo.lookup_reference(ref)
refs[m.group(1)] = reference.hex
return refs
def create_branch(self, name, target):
r"""
Create new branch.
"""
if not is_valid_value(name):
raise InvalidParamException("name is required")
if not is_valid_hex(target):
raise InvalidParamException("target is required")
target = sha_hex2bin(target)
try:
self.repo.create_reference('refs/heads/%s' % name, target)
except Exception, e:
raise InvalidParamException(str(e))
return True
class GitRepo:
def __init__(self, repo_path):
self.repo = Repository(repo_path)
def checkout_by(self, commit_id):
ref = 'refs/tags/t-%s' % commit_id
if self.repo.references.get(ref) is None:
self.repo.create_reference(ref, commit_id)
self.repo.checkout(ref)
self.repo.references[ref].delete()
def master(self):
branch = self.repo.lookup_branch('master')
self.repo.checkout(branch)
def get_all_commit_id(self):
self.master()
return self.repo.walk(self.repo.head.target,
GIT_SORT_TOPOLOGICAL | GIT_SORT_REVERSE)
def _fast_forward(self, local_path, merge_target, branch):
# fast-forward all the branches.
# pygit2 repo
repo = Repository(discover_repository(local_path))
# convert merge_target from hex into oid.
fetch_head = repo.revparse_single(merge_target)
# try to resolve a common anscestor between fetched and local
try:
head = repo.revparse_single(branch)
except KeyError:
# Doesn't exist. Create and done.
repo.create_reference(branch, fetch_head.oid)
return True, 'Created new branch: %s' % branch
if head.oid == fetch_head.oid:
return True, 'Source and target are identical.'
# raises KeyError if no merge bases found.
common_oid = repo.merge_base(head.oid, fetch_head.oid)
# Three different outcomes between the remaining cases.
if common_oid.hex not in (head.oid.hex, fetch_head.oid.hex):
# common ancestor is beyond both of these, not going to
# attempt a merge here and will assume this:
return False, 'Branch will diverge.'
elif common_oid.hex == fetch_head.oid.hex:
# Remote is also the common ancestor, so nothing to do.
return True, 'No new changes found.'
# This case remains: common_oid.hex == head.oid.hex, meaning
# this local repository is the ancestor of further changes
# fetched from the remote - remote newer, so fast-forward.
ref = repo.lookup_reference(branch)
ref.delete()
repo.create_reference(branch, fetch_head.oid)
return True, 'Fast-forwarded branch: %s' % branch
class DictRepository(object):
"""The :class:`DictRepository <DictRepository>` object.
:param repo_or_path:
The path to a repository, or an existing pygit2.Repository object.
If it is a path that does not exist, a new bare git repository will
be initialized there. If it is a path that does exist, then the
directory will be used as a bare git repository.
:type repo_or_path: string or pygit2.Repository
"""
def __init__(self, repo_or_path=None):
self._default_author = get_default_author()
if isinstance(repo_or_path, Repository):
self._repo = repo_or_path
elif os.path.isdir(repo_or_path):
self._repo = Repository(repo_or_path)
else:
self._repo = init_repository(repo_or_path, True) # bare repo
def _key_to_ref(self, key):
return "refs/%s/HEAD" % key
def get_commit_oid_for_key(self, key):
return self._repo[self._repo.lookup_reference(self._key_to_ref(key)).oid].oid
def get_raw_dict_for_commit_oid(self, commit_oid):
return json.loads(self._repo[self._repo[commit_oid].tree[DATA].oid].data)
def get_parent_oids_for_commit_oid(self, commit_oid):
return [parent.oid for parent in self._repo[commit_oid].parents]
def raw_commit(self, key, raw_dict, author, committer, message, parents):
"""Commit a dict to this :class:`DictRepository <DictRepository>`.
It is recommended that you use the :class:`GitDict <GitDict>` commit
method instead.
:param raw_dict: the data to commit.
:type raw_dict: dict
:param author:
The author of the commit. If None, will be replaced with default.
:type author: pygit2.Signature
:param committer:
The committer of this commit. If None, will be replaced with author.
:type committer: pygit2.Signature
:param message: The commit message.
:type message: string
:param parents:
A list of 20-byte object IDs of parent commits. An empty list
means this is the first commit.
:return: The oid of the new commit.
:rtype: 20 bytes
"""
if not isinstance(raw_dict, dict):
raise ValueError("%s is not a dict" % raw_dict)
author = author or self._default_author.signature()
committer = committer or author
blob_id = self._repo.write(GIT_OBJ_BLOB, json.dumps(raw_dict))
# TreeBuilder doesn't support inserting into trees, so we roll our own
tree_id = self._repo.write(GIT_OBJ_TREE, "100644 %s\x00%s" % (DATA, blob_id))
return self._repo.create_commit(self._key_to_ref(key), author, committer, message, tree_id, parents)
def create(self, key, dict={}, autocommit=False, message="first commit", author=None, committer=None):
"""Create a new :class:`GitDict <GitDict>`
:param key: The key of the new :class:`GitDict <GitDict>`
:type key: :class:`GitDict <GitDict>`
:param dict: (optional) The value of the dict. Defaults to empty.
:type dict: dict
:param autocommit:
(optional) Whether the :class:`GitDict <GitDict>` should
automatically commit. Defaults to false.
:type autocommit: boolean
:param message:
(optional) Message for first commit. Defaults to "first commit".
:type message: string
:param author:
(optional) The signature for the author of the first commit.
Defaults to global author.
:type author: pygit2.Signature
:param committer:
(optional) The signature for the committer of the first commit.
Defaults to author.
:type author: pygit2.Signature
:returns: the GitDict
:rtype: :class:`GitDict <GitDict>`
"""
self.raw_commit(key, dict, author, committer, message, [])
return self.get(key, autocommit=autocommit)
def has(self, key):
"""Determine whether there is an entry for key in this repository.
:param key: The key to check
:type key: string
:returns: whether there is an entry
:rtype: boolean
"""
try:
self._repo.lookup_reference(self._key_to_ref(key))
return True
except KeyError:
return False
def get(self, key, autocommit=False):
"""Obtain the :class:`GitDict <GitDict>` for a key.
:param key: The key to look up.
:type key: string
:param autocommit:
(optional) Whether the :class:`GitDict <GitDict>` should
automatically commit. Defaults to false.
:type autocommit: boolean
:returns: the GitDict
:rtype: :class:`GitDict <GitDict>`
:raises: KeyError if there is no entry for key
"""
return GitDict(self, key, autocommit=autocommit)
def fast_forward(self, from_dict, to_dict):
"""Fast forward a :class:`GitDict <GitDict>`.
:param from_dict: the :class:`GitDict <GitDict>` to fast forward.
:type from_dict: :class:`GitDict <GitDict>`
:param to_dict: the :class:`GitDict <GitDict>`to fast forward to.
:type to_dict: :class:`GitDict <GitDict>`
"""
from_ref = self._key_to_ref(from_dict.key)
self._repo.lookup_reference(from_ref).delete()
self._repo.create_reference(from_ref, self.get_commit_oid_for_key(to_dict.key))
def clone(self, original, key):
"""Clone a :class:`GitDict <GitDict>`.
:param original: the :class:`GitDict <GitDict>` to clone
:type original: :class:`GitDict <GitDict>`
:param key: where to clone to
:type key: string
:raises: ValueError if to_key already exists.
"""
try:
self._repo.create_reference(self._key_to_ref(key), self.get_commit_oid_for_key(original.key))
return self.get(key, autocommit=original.autocommit)
except GitError:
raise ValueError("Cannot clone to %s, there is already a dict there." % key)
def _checkout_commit(self, repo: pygit2.Repository, commit):
repo.create_reference(DataExtractor.WORKING_TAG_REFNAME, commit.id)
repo.checkout(DataExtractor.WORKING_TAG_REFNAME)
repo.lookup_reference(DataExtractor.WORKING_TAG_REFNAME).delete()
class Document(object):
'''
Class representing a document, interacts with the git
database
'''
def __init__( self, name, create=False, rootPath=None ):
'''
Constructor
Args:
name The name of the document
create If true, will create a document
rootPath The rootPath to use (if not supplied, uses default)
Exceptions:
RepoNotFound if repository isn't found
'''
if not rootPath:
rootPath = DEFAULT_ROOT_PATH
targetDir = os.path.join( rootPath, name + '.git' )
if create:
# Create a bare repository
self.repo = init_repository( targetDir, True )
self._CreateMasterBranch()
else:
try:
self.repo = Repository( targetDir )
except KeyError:
raise RepoNotFound()
def _CreateMasterBranch( self ):
'''
Creates the master branch on the repo w/ default file.
For now this is just a file named layout
'''
commitId = CommitBlob(
self.repo, '', SECTION_INDEX_FILENAME, 'Initial commit'
)
self.repo.create_reference( MASTER_REF, commitId )
@staticmethod
def _IsSectionRef( refName ):
'''
Checks if a refererence name refers to a section
Args:
refName: The reference name
Returns:
A boolean
'''
return refName.startswith( SECTION_REF_PREFIX )
@staticmethod
def _RefNameToSectionName( refName ):
'''
Converts a reference name to a section name
Args:
ref: The reference name
'''
return refName[ len(SECTION_REF_PREFIX) : ]
def _SectionRefs( self ):
'''
Gets an iterator over the section refs
'''
return (
(
self._RefNameToSectionName( ref ),
self.repo.lookup_reference( ref ),
)
for ref in self.repo.listall_references()
if self._IsSectionRef( ref )
)
def Sections( self ):
'''
Gets an iterator over all the sections
'''
return (
Section( name, self.repo[ref.oid], self.repo )
for name, ref in self._SectionRefs()
)
def CurrentSections( self ):
'''
Gets the current sections with their positions
Returns:
A list of tuples ( position, section )
'''
return enumerate( self._CurrentSections() )
def _CurrentSections( self ):
'''
Internal method to get the current sections
in order, without position numbers
Returns:
An iterator over the sections
'''
index = SectionIndex(self.repo)
for s in index.CurrentSections():
yield self.FindSection(s.name)
def FindSection( self, name ):
'''
Finds a section by name
Args:
name The name of the section to find
Returns:
The section if found
Exceptions:
SectionNotFound if section not found
'''
try:
ref = self.repo.lookup_reference(
SECTION_REF_PREFIX + name
)
except KeyError:
raise SectionNotFound()
return Section( name, self.repo[ref.oid], self.repo )
def AddSection( self, name, content='' ):
'''
Creates a new section
Args:
name The name of the section
content The optional initial content of the
section
Returns:
The new Section object
'''
# TODO: Should probably make
# sure no such section exists
commitId = CommitBlob(
self.repo,
content,
name,
'Created section ' + name
)
ref = self.repo.create_reference(
SECTION_REF_PREFIX + name,
commitId
)
index = SectionIndex(self.repo)
index.AddSection( name )
index.Save( self.repo )
return Section( name, self.repo[ ref.oid ], self.repo )
def RemoveSection( self, name ):
'''
Removes a section.
This function does not actually delete the data associated
with a section, it just removes it from the index.
Args:
name The name of the section to remove
'''
index = SectionIndex( self.repo )
index.RemoveSection( name )
index.Save( self.repo )
