def __init__(self,
remote_url,
repo_path,
mount_path,
credentials,
current_path="current",
history_path="history",
branch=None,
user="******",
group="root",
**kwargs):
"""
Clone repo from a remote into repo_path/<repo_name> and checkout to
a specific branch.
:param str remote_url: URL of the repository to clone
:param str repo_path: Where are all the repos are cloned
:param str branch: Branch to checkout after the
clone. The default is to use the remote's default branch.
"""
self.remote_url = remote_url
self.repo_path = repo_path
self.mount_path = mount_path
self.current_path = current_path
self.history_path = history_path
self.branch = branch
self.routes = []
log.info('Cloning into {}'.format(self.repo_path))
self.repo = Repository.clone(self.remote_url, self.repo_path,
self.branch, credentials)
log.info('Done cloning')
self.repo.credentials = credentials
submodules = os.path.join(self.repo_path, '.gitmodules')
ignore = os.path.join(self.repo_path, '.gitignore')
self.repo.ignore = CachedIgnore(submodules=submodules,
ignore=ignore,
exclude=kwargs['ignore_file'] or None,
hard_ignore=kwargs['hard_ignore'])
self.uid = getpwnam(user).pw_uid
self.gid = getgrnam(group).gr_gid
self.commit_queue = kwargs['commit_queue']
self.mount_time = int(time.time())
self.max_size = kwargs['max_size']
self.max_offset = kwargs['max_offset']
try:
self.repo.commits.update()
except Exception, e:
log.error("[Exception] repo.commits.update failed: %s", str(e))
sys.exit()
def on_idle(self):
"""
On idle, we have 4 cases:
1. We have to commit and also need to merge some commits from remote.
In this case, we commit and announce ourself for merging
2. We are behind from remote, so we announce for merging
3. We only need to commit
4. We announced for merging and nobody is writing in this momement.
In this case we are safe to merge and push.
"""
if not syncing.is_set():
log.debug("Set syncing event (%d pending writes)", writers.value)
syncing.set()
else:
log.debug("Idling (%d pending writes)", writers.value)
if writers.value == 0:
if self.commits:
log.info("Get some commits")
self.commit(self.commits)
self.commits = []
count = 0
log.debug("Start syncing, first attempt.")
while not self.sync() and count < 5:
fuzz = random.randint(0, 1000) / 1000
wait = 2**count + fuzz
log.debug("Failed to sync. Going to sleep for %d seconds",
wait)
time.sleep(wait)
count += 1
log.debug("Retry-ing to sync with remote. Attempt #%d", count)
if count >= 5:
log.error("Didn't manage to sync, I need some help")
def clone(cls, remote_url, path, branch=None, credentials=None):
"""Clone a repo in a give path and update the working directory with
a checkout to head (GIT_CHECKOUT_SAFE_CREATE)
:param str remote_url: URL of the repository to clone
:param str path: Local path to clone into
:param str branch: Branch to checkout after the
clone. The default is to use the remote's default branch.
"""
try:
repo = clone_repository(remote_url,
path,
checkout_branch=branch,
callbacks=credentials)
except Exception as e:
log.error("Error on cloning the repository: ", exc_info=True)
repo.checkout_head()
return cls(repo)
def on_idle(self):
"""
On idle, we have 4 cases:
1. We have to commit and also need to merge some commits from remote.
In this case, we commit and announce ourself for merging
2. We are behind from remote, so we announce for merging
3. We only need to commit
4. We announced for merging and nobody is writing in this momement.
In this case we are safe to merge and push.
"""
if not syncing.is_set():
log.debug("Set syncing event (%d pending writes)", writers.value)
syncing.set()
else:
log.debug("Idling (%d pending writes)", writers.value)
if writers.value == 0:
if self.commits:
log.info("Get some commits")
self.commit(self.commits)
self.commits = []
count = 0
log.debug("Start syncing, first attempt.")
while not self.sync() and count < 5:
fuzz = random.randint(0, 1000) / 1000
wait = 2 ** count + fuzz
log.debug("Failed to sync. Going to sleep for %d seconds", wait)
time.sleep(wait)
count += 1
log.debug("Retry-ing to sync with remote. Attempt #%d", count)
if count >= 5:
log.error("Didn't manage to sync, I need some help")
class Repository(object):
def __init__(self, repository, commits=None):
self._repo = repository
self.commits = commits or CommitCache(self)
self.behind = False
def __getitem__(self, item):
"""
Proxy method for pygit2.Repository
"""
return self._repo[item]
def __getattr__(self, attr):
"""
Proxy method for pygit2.Repository
"""
if attr not in self.__dict__:
return getattr(self._repo, attr)
else:
return self.__dict__[attr]
def ahead(self, upstream, branch):
ahead, _ = self.diverge(upstream, branch)
return ahead
def diverge(self, upstream, branch):
reference = "{}/{}".format(upstream, branch)
remote_branch = self.lookup_branch(reference, GIT_BRANCH_REMOTE)
local_branch = self.lookup_branch(branch, GIT_BRANCH_LOCAL)
if remote_branch.target == local_branch.target:
return False, False
diverge_commits = self.find_diverge_commits(local_branch,
remote_branch)
behind = len(diverge_commits.second_commits) > 0
ahead = len(diverge_commits.first_commits) > 0
return ahead, behind
def checkout(self, ref, *args, **kwargs):
result = self._repo.checkout(ref, *args, **kwargs)
# update ignore cache after a checkout
self.ignore.update()
status = self._repo.status()
for path, status in iteritems(status):
# path is in current status, move on
if status == GIT_STATUS_CURRENT:
continue
# check if file exists or not
full_path = self._full_path(path)
if path not in self._repo.index:
if path not in self.ignore:
try:
os.unlink(full_path)
except OSError:
# path points to a directory containing untracked files
rmtree(
full_path,
onerror=lambda function, fpath, excinfo: log.info(
"Repository: Checkout couldn't delete %s", fpath
)
)
continue
# check files stats
stats = self.get_git_object_default_stats(ref, path)
current_stat = os.lstat(full_path)
if stats['st_mode'] != current_stat.st_mode:
try:
os.chmod(full_path, current_stat.st_mode)
except OSError:
log.info("Repository: Checkout couldn't chmod %s",
full_path)
self._repo.index.add(self._sanitize(path))
return result
def _sanitize(self, path):
if path is not None and path.startswith("/"):
path = path[1:]
return path
def push(self, upstream, branch, credentials):
""" Push changes from a branch to a remote
Examples::
repo.push("origin", "master")
"""
remote = self.get_remote(upstream)
remote.push(["refs/heads/%s" % (branch)], callbacks=credentials)
def fetch(self, upstream, branch_name, credentials):
"""
Fetch from remote and return True if we are behind or False otherwise
"""
remote = self.get_remote(upstream)
remote.fetch(callbacks=credentials)
_, behind = self.diverge(upstream, branch_name)
self.behind = behind
return behind
def commit(self, message, author, commiter, parents=None, ref="HEAD"):
""" Wrapper for create_commit. It creates a commit from a given ref
(default is HEAD)
"""
status = self._repo.status()
if status == {}:
return None
# sign the author
author = Signature(author[0], author[1])
commiter = Signature(commiter[0], commiter[1])
# write index localy
tree = self._repo.index.write_tree()
self._repo.index.write()
# get parent
if parents is None:
parents = [self._repo.revparse_single(ref).id]
return self._repo.create_commit(ref, author, commiter, message,
tree, parents)
@classmethod
def clone(cls, remote_url, path, branch=None, credentials=None):
"""Clone a repo in a give path and update the working directory with
a checkout to head (GIT_CHECKOUT_SAFE_CREATE)
:param str remote_url: URL of the repository to clone
:param str path: Local path to clone into
:param str branch: Branch to checkout after the
clone. The default is to use the remote's default branch.
"""
hasExistingRepo = False
try:
existingRepoPath = discover_repository(path)
if existingRepoPath<>"":
hasExistingRepo = True
except Exception, e:
log.debug("[Exception] discover_repository repo not found: %s", str(e))
pass
if hasExistingRepo == False:
log.debug("clone_repository %s", path)
try:
repo = clone_repository(remote_url, path, checkout_branch=branch,
callbacks=credentials)
except Exception, e:
log.error("[Exception] clone_repository failed: %s", str(e))
sys.exit()
repo.checkout_head()
log.info("repo cloned")
try:
repo = clone_repository(remote_url, path, checkout_branch=branch,
callbacks=credentials)
except Exception, e:
log.error("[Exception] clone_repository failed: %s", str(e))
sys.exit()
repo.checkout_head()
log.info("repo cloned")
else:
log.debug("init_repository %s", existingRepoPath)
try:
repo = init_repository(existingRepoPath)
except Exception, e:
log.error("[Exception] init_repository failed: %s", str(e))
sys.exit()
log.info("existing repo '%s' opened", existingRepoPath)
return cls(repo)
def _is_searched_entry(self, entry_name, searched_entry, path_components):
"""
Checks if a tree entry is the one that is being searched for. For
that, the name has to correspond and it has to be the last element
in the path_components list (this means that the path corresponds
exactly).
:param entry_name: the name of the tree entry
:param searched_entry: the name of the object that is being searched