def sync(self): log.debug("Check if I'm ahead") need_to_push = self.repository.ahead(self.upstream, self.branch) sync_done.clear() if self.repository.behind: log.debug("I'm behind so I start merging") try: self.merge() need_to_push = True except: log.exception("Merge failed") return if need_to_push: try: with remote_operation: log.debug("Start pushing") self.repository.push(self.upstream, self.branch) self.repository.behind = False log.info("Push done") log.debug("Clear syncing") syncing.clear() log.debug("Set sync_done") sync_done.set() log.debug("Set push_successful") push_successful.set() except: push_successful.clear() fetch.set() log.exception("Push failed") else: sync_done.set() syncing.clear()
def __init__(self, remote_url, repo_path, mount_path, credentials, current_path="current", history_path="history", branch=None, user="******", group="root", **kwargs): """ Clone repo from a remote into repo_path/<repo_name> and checkout to a specific branch. :param str remote_url: URL of the repository to clone :param str repo_path: Where are all the repos are cloned :param str branch: Branch to checkout after the clone. The default is to use the remote's default branch. """ self.remote_url = remote_url self.repo_path = repo_path self.mount_path = mount_path self.current_path = current_path self.history_path = history_path self.branch = branch self.routes = [] log.info("Cloning into {}".format(self.repo_path)) self.repo = Repository.clone(self.remote_url, self.repo_path, self.branch, credentials) log.info("Done cloning") self.repo.credentials = credentials submodules = os.path.join(self.repo_path, ".gitmodules") ignore = os.path.join(self.repo_path, ".gitignore") self.repo.ignore = CachedIgnore( submodules=submodules, ignore=ignore, exclude=kwargs["ignore_file"] or None, hard_ignore=kwargs["hard_ignore"], ) self.uid = getpwnam(user).pw_uid self.gid = getgrnam(group).gr_gid self.commit_queue = kwargs["commit_queue"] self.mount_time = int(time.time()) self.max_size = kwargs["max_size"] self.max_offset = kwargs["max_offset"] self.repo.commits.update() self.workers = []
def on_idle(self): """ On idle, we have 4 cases: 1. We have to commit and also need to merge some commits from remote. In this case, we commit and announce ourself for merging 2. We are behind from remote, so we announce for merging 3. We only need to commit 4. We announced for merging and nobody is writing in this momement. In this case we are safe to merge and push. """ if not syncing.is_set(): log.debug("Set syncing event (%d pending writes)", writers.value) syncing.set() else: log.debug("Idling (%d pending writes)", writers.value) if writers.value == 0: if self.commits: log.info("Get some commits") self.commit(self.commits) self.commits = [] count = 0 log.debug("Start syncing") while not self.sync(): if count < 5: count += 1 fuzz = random.randint(0, 1000) / 1000 wait = 2**count + fuzz log.debug("Failed. Going to sleep for %d seconds", wait) time.sleep(wait) log.debug("Retry-ing")
def on_idle(self): """ On idle, we have 4 cases: 1. We have to commit and also need to merge some commits from remote. In this case, we commit and announce ourself for merging 2. We are behind from remote, so we announce for merging 3. We only need to commit 4. We announced for merging and nobody is writing in this momement. In this case we are safe to merge and push. """ if not syncing.is_set(): log.debug("Set syncing event (%d pending writes)", writers.value) syncing.set() else: log.debug("Idling (%d pending writes)", writers.value) if writers.value == 0: if self.commits: log.info("Get some commits") self.commit(self.commits) self.commits = [] count = 0 log.debug("Start syncing") while not self.sync(): if count < 5: count += 1 fuzz = random.randint(0, 1000) / 1000 wait = 2 ** count + fuzz log.debug("Failed. Going to sleep for %d seconds", wait) time.sleep(wait) log.debug("Retry-ing")
def mkdir(self, path, mode, gitignore): result = super(CurrentView, self).mkdir(path, mode) if gitignore == True: log.debug("[ignore] mkdir:%s", path) return result keep_path = "{}/.keep".format(path) full_path = self.repo._full_path(keep_path) if not os.path.exists(keep_path): global writers fh = os.open(full_path, os.O_WRONLY | os.O_CREAT) writers += 1 log.info("CurrentView: Open %s for write", full_path) super(CurrentView, self).chmod(keep_path, 0o644) self.dirty[fh] = { 'message': "Create the {} directory".format(path), 'stage': True } self.release(keep_path, fh) log.debug("CurrentView: Created directory %s with mode %s", path, mode) return result
def __init__(self, remote_url, repo_path, mount_path, credentials, current_path="current", history_path="history", branch=None, user="******", group="root", **kwargs): """ Clone repo from a remote into repo_path/<repo_name> and checkout to a specific branch. :param str remote_url: URL of the repository to clone :param str repo_path: Where are all the repos are cloned :param str branch: Branch to checkout after the clone. The default is to use the remote's default branch. """ self.remote_url = remote_url self.repo_path = repo_path self.mount_path = mount_path self.current_path = current_path self.history_path = history_path self.branch = branch self.routes = [] log.info('Cloning into {}'.format(self.repo_path)) self.repo = Repository.clone(self.remote_url, self.repo_path, self.branch, credentials) log.info('Done cloning') self.repo.credentials = credentials submodules = os.path.join(self.repo_path, '.gitmodules') ignore = os.path.join(self.repo_path, '.gitignore') self.repo.ignore = CachedIgnore(submodules=submodules, ignore=ignore, exclude=kwargs['ignore_file'] or None, hard_ignore=kwargs['hard_ignore']) self.uid = getpwnam(user).pw_uid self.gid = getgrnam(group).gr_gid self.commit_queue = kwargs['commit_queue'] self.mount_time = int(time.time()) self.max_size = kwargs['max_size'] self.max_offset = kwargs['max_offset'] try: self.repo.commits.update() except Exception, e: log.error("[Exception] repo.commits.update failed: %s", str(e)) sys.exit()
def run(self): while True: fetch.wait(self.timeout) if shutting_down.is_set(): log.info("Stop fetch worker") break self.fetch()
def work(self): while True: fetch.wait(self.timeout) if shutting_down.is_set(): log.info("Stop fetch worker") break self.fetch()
def destroy(self, path): log.debug('Stopping workers') shutting_down.set() fetch.set() for worker in self.workers: worker.join() log.debug('Workers stopped') #shutil.rmtree(self.repo_path) log.info('Successfully umounted %s', self.mount_path)
def destroy(self, path): log.debug('Stopping workers') shutting_down.set() fetch.set() for worker in self.workers: worker.join() log.debug('Workers stopped') shutil.rmtree(self.repo_path) log.info('Successfully umounted %s', self.mount_path)
def get_credentials(args): if args.password: credentials = UserPass(args.username, args.password) elif args.ssh_agent: log.info('Using SSH agent with remote user: %s', args.ssh_user) credentials = KeypairFromAgent(args.ssh_user) else: log.info('Using SSH user: %s, key: %s', args.ssh_user, args.ssh_key) credentials = Keypair(args.ssh_user, args.ssh_key + ".pub", args.ssh_key, "") return RemoteCallbacks(credentials=credentials)
def work(self): while True: timeout = self.timeout if idle.is_set(): timeout = self.idle_timeout fetch.wait(timeout) if shutting_down.is_set(): log.info("Stop fetch worker") break self.fetch()
def work(self): while True: timeout = self.timeout if idle.is_set(): timeout = self.idle_timeout log.debug("Wait for {}".format(timeout)) fetch.wait(timeout) if shutting_down.is_set(): log.info("Stop fetch worker") break self.fetch()
def run(self): while True: if shutting_down.is_set(): log.info("Stop sync worker") break try: job = self.commit_queue.get(timeout=self.timeout, block=True) if job['type'] == 'commit': self.commits.append(job) log.debug("Got a commit job") except Empty: log.debug("Nothing to do right now, going idle") self.on_idle()
def __init__(self, remote_url, repo_path, mount_path, credentials, current_path="current", history_path="history", branch=None, user="******", group="root", **kwargs): """ Clone repo from a remote into repo_path/<repo_name> and checkout to a specific branch. :param str remote_url: URL of the repository to clone :param str repo_path: Where are all the repos are cloned :param str branch: Branch to checkout after the clone. The default is to use the remote's default branch. """ self.remote_url = remote_url self.repo_path = repo_path self.mount_path = mount_path self.current_path = current_path self.history_path = history_path self.branch = branch self.routes = [] log.info('Cloning into {}'.format(self.repo_path)) self.repo = Repository.clone(self.remote_url, self.repo_path, self.branch, credentials) log.info('Done cloning') self.repo.credentials = credentials submodules = os.path.join(self.repo_path, '.gitmodules') ignore = os.path.join(self.repo_path, '.gitignore') self.repo.ignore = CachedIgnore(submodules=submodules, ignore=ignore, exclude=kwargs['ignore_file'] or None, hard_ignore=kwargs['hard_ignore']) self.uid = getpwnam(user).pw_uid self.gid = getgrnam(group).gr_gid self.commit_queue = kwargs['commit_queue'] self.mount_time = int(time.time()) self.max_size = kwargs['max_size'] self.max_offset = kwargs['max_offset'] self.repo.commits.update() self.workers = []
def work(self): while True: if shutting_down.is_set(): log.info("Stop sync worker") break try: job = self.commit_queue.get(timeout=self.timeout, block=True) if job['type'] == 'commit': self.commits.append(job) log.debug("Got a commit job") except Empty: log.debug("Nothing to do right now, going idle") self.on_idle()
def fetch(self): with remote_operation: fetch.clear() try: log.debug("Start fetching") was_behind = self.repository.fetch(self.upstream, self.branch) fetch_successful.set() if was_behind: log.info("Fetch done") else: log.debug("Nothing to fetch") except: fetch_successful.clear() log.exception("Fetch failed")
def access(self, path, mode): if getattr(self, 'date', None): log.info('PATH: %s', path) if path == '/': available_dates = self.repo.get_commit_dates() if self.date not in available_dates: raise FuseOSError(ENOENT) else: commits = self.repo.get_commits_by_date(self.date) dirname = os.path.split(path)[1] if dirname not in commits: raise FuseOSError(ENOENT) else: if path != '/': raise FuseOSError(ENOENT) return 0
def sync(self): log.debug("Check if I'm ahead") need_to_push = self.repository.ahead(self.upstream, self.branch) sync_done.clear() if self.repository.behind: log.debug("I'm behind so I start merging") try: log.debug("Start fetching") self.repository.fetch(self.upstream, self.branch, self.credentials) log.debug("Done fetching") log.debug("Start merging") self.merge() log.debug("Merge done with success, ready to push") need_to_push = True except: log.exception("Merge failed") return False if need_to_push: try: with remote_operation: log.debug("Start pushing") self.repository.push(self.upstream, self.branch, self.credentials) self.repository.behind = False log.info("Push done") log.debug("Clear syncing") syncing.clear() log.debug("Set sync_done") sync_done.set() log.debug("Set push_successful") push_successful.set() except Exception as error: push_successful.clear() fetch.set() log.debug("Push failed because of %s", error) return False else: log.debug("Sync done, clearing") sync_done.set() syncing.clear() return True
def solve_conflicts(self, conflicts): if conflicts: for common, theirs, ours in conflicts: if not ours and theirs: log.debug("AcceptMine: if we deleted the file and they " "didn't, remove the file") self.repository.index.remove(theirs.path) elif ours and not theirs: log.debug("AcceptMine: if they deleted the file and we " "didn't, add the file") self.repository.index.add(ours.path) else: log.debug("AcceptMine: overwrite all file with our " "content") with open(self.repository._full_path(ours.path), "w") as f: f.write(self.repository.get(ours.id).data) self.repository.index.add(ours.path) else: log.info("AcceptMine: No conflicts to solve")
def __init__(self, remote_url, repo_path, mount_path, credentials, branch=None, user="******", group="root", **kwargs): """ Clone repo from a remote into repo_path/<repo_name> and checkout to a specific branch. :param str remote_url: URL of the repository to clone :param str repo_path: Where are all the repos are cloned :param str branch: Branch to checkout after the clone. The default is to use the remote's default branch. """ self.remote_url = remote_url self.repo_path = repo_path self.mount_path = mount_path self.branch = branch self.routes = [] log.info('Cloning into %s' % self.repo_path) self.repo = Repository.clone(self.remote_url, self.repo_path, self.branch, credentials) log.info('Done cloning') self.repo.credentials = credentials self.repo.ignore = CachedIgnore(submodules=True, ignore=True, path=self.repo_path) self.uid = getpwnam(user).pw_uid self.gid = getgrnam(group).gr_gid self.commit_queue = kwargs['commit_queue'] self.mount_time = int(time.time()) self.max_size = kwargs['max_size'] self.max_offset = kwargs['max_offset'] self.repo.commits.update() self.workers = []
def checkout(self, ref, *args, **kwargs): result = self._repo.checkout(ref, *args, **kwargs) # update ignore cache after a checkout self.ignore.update() status = self._repo.status() for path, status in iteritems(status): # path is in current status, move on if status == GIT_STATUS_CURRENT: continue # check if file exists or not full_path = self._full_path(path) if path not in self._repo.index: if path not in self.ignore: try: os.unlink(full_path) except OSError: # path points to a directory containing untracked files rmtree( full_path, onerror=lambda function, fpath, excinfo: log.info( "Repository: Checkout couldn't delete %s", fpath ) ) continue # check files stats stats = self.get_git_object_default_stats(ref, path) current_stat = os.lstat(full_path) if stats['st_mode'] != current_stat.st_mode: try: os.chmod(full_path, current_stat.st_mode) except OSError: log.info("Repository: Checkout couldn't chmod %s", full_path) self._repo.index.add(self._sanitize(path)) return result
def mkdir(self, path, mode): result = super(CurrentView, self).mkdir(path, mode) keep_path = "%s/.keep" % path full_path = self.repo._full_path(keep_path) if not os.path.exists(keep_path): global writers fh = os.open(full_path, os.O_WRONLY | os.O_CREAT) writers += 1 log.info("CurrentView: Open %s for write", full_path) super(CurrentView, self).chmod(keep_path, 0o644) self.dirty[fh] = { 'message': "Create the %s directory" % path, } self.release(keep_path, fh) log.debug("CurrentView: Created directory %s with mode %s", path, mode) return result
def work(self): idle_times = 0 while True: if shutting_down.is_set(): log.info("Stop sync worker") break try: job = self.commit_queue.get(timeout=self.timeout, block=True) if job['type'] == 'commit': self.commits.append(job) log.debug("Got a commit job") idle_times = 0 idle.clear() except Empty: log.debug("Nothing to do right now, going idle") if idle_times > self.min_idle_times: idle.set() idle_times += 1 self.on_idle()
def work(self): idle_times = 0 while True: if shutting_down.is_set(): log.info("Stop sync worker") break try: job = self.commit_queue.get(timeout=self.timeout, block=True) if job["type"] == "commit": self.commits.append(job) log.debug("Got a commit job") idle_times = 0 idle.clear() except Empty: log.debug("Nothing to do right now, going idle") if idle_times > self.min_idle_times: idle.set() idle_times += 1 self.on_idle()
def on_idle(self): """ On idle, we have 4 cases: 1. We have to commit and also need to merge some commits from remote. In this case, we commit and announce ourself for merging 2. We are behind from remote, so we announce for merging 3. We only need to commit 4. We announced for merging and nobody is writing in this momement. In this case we are safe to merge and push. """ if not syncing.is_set(): log.debug("Set syncing event (%d pending writes)", writers.value) syncing.set() else: log.debug("Idling (%d pending writes)", writers.value) if writers.value == 0: if self.commits: log.info("Get some commits") self.commit(self.commits) self.commits = [] log.debug("Start syncing") self.sync()
callbacks=credentials) except Exception, e: log.error("[Exception] clone_repository failed: %s", str(e)) sys.exit() repo.checkout_head() log.info("repo cloned") else: log.debug("init_repository %s", existingRepoPath) try: repo = init_repository(existingRepoPath) except Exception, e: log.error("[Exception] init_repository failed: %s", str(e)) sys.exit() log.info("existing repo '%s' opened", existingRepoPath) return cls(repo) def _is_searched_entry(self, entry_name, searched_entry, path_components): """ Checks if a tree entry is the one that is being searched for. For that, the name has to correspond and it has to be the last element in the path_components list (this means that the path corresponds exactly). :param entry_name: the name of the tree entry :param searched_entry: the name of the object that is being searched for :type searched_entry: str :param path_components: the path of the object being searched for
def open_for_read(self, path, flags): full_path = self.repo._full_path(path) log.info("CurrentView: Open %s for read", path) return os.open(full_path, flags)
def prepare_components(args): commit_queue = CommitQueue() credentials = get_credentials(args) try: # setting router router = Router(remote_url=args.remote_url, mount_path=args.mount_point, current_path=args.current_path, history_path=args.history_path, repo_path=args.repo_path, branch=args.branch, user=args.user, group=args.group, max_size=args.max_size * 1024 * 1024, max_offset=args.max_size * 1024 * 1024, commit_queue=commit_queue, credentials=credentials, ignore_file=args.ignore_file, hard_ignore=args.hard_ignore) except KeyError as error: sys.stderr.write( "Can't clone reference origin/%s from remote %s: %s\n" % (args.branch, args.remote_url, error)) raise error # register all the routes routes = prepare_routes(args) router.register(routes) # determine global git user/email if not set (default is to use hostname) if ('@%s' % socket.gethostname() ) in args.commiter_email and router.repo.default_signature: sig = router.repo.default_signature args.commiter_email = sig.email args.commiter_name = sig.name log.info('Using default signature: %s <%s>', sig.name, sig.email) # setup workers merge_worker = SyncWorker(args.commiter_name, args.commiter_email, args.commiter_name, args.commiter_email, commit_queue=commit_queue, repository=router.repo, upstream="origin", branch=args.branch, repo_path=router.repo_path, timeout=args.merge_timeout, credentials=credentials, min_idle_times=args.min_idle_times) fetch_worker = FetchWorker(upstream="origin", branch=args.branch, repository=router.repo, timeout=args.fetch_timeout, credentials=credentials, idle_timeout=args.idle_fetch_timeout) merge_worker.daemon = True fetch_worker.daemon = True router.workers = [merge_worker, fetch_worker] return merge_worker, fetch_worker, router
class Repository(object): def __init__(self, repository, commits=None): self._repo = repository self.commits = commits or CommitCache(self) self.behind = False def __getitem__(self, item): """ Proxy method for pygit2.Repository """ return self._repo[item] def __getattr__(self, attr): """ Proxy method for pygit2.Repository """ if attr not in self.__dict__: return getattr(self._repo, attr) else: return self.__dict__[attr] def ahead(self, upstream, branch): ahead, _ = self.diverge(upstream, branch) return ahead def diverge(self, upstream, branch): reference = "{}/{}".format(upstream, branch) remote_branch = self.lookup_branch(reference, GIT_BRANCH_REMOTE) local_branch = self.lookup_branch(branch, GIT_BRANCH_LOCAL) if remote_branch.target == local_branch.target: return False, False diverge_commits = self.find_diverge_commits(local_branch, remote_branch) behind = len(diverge_commits.second_commits) > 0 ahead = len(diverge_commits.first_commits) > 0 return ahead, behind def checkout(self, ref, *args, **kwargs): result = self._repo.checkout(ref, *args, **kwargs) # update ignore cache after a checkout self.ignore.update() status = self._repo.status() for path, status in iteritems(status): # path is in current status, move on if status == GIT_STATUS_CURRENT: continue # check if file exists or not full_path = self._full_path(path) if path not in self._repo.index: if path not in self.ignore: try: os.unlink(full_path) except OSError: # path points to a directory containing untracked files rmtree( full_path, onerror=lambda function, fpath, excinfo: log.info( "Repository: Checkout couldn't delete %s", fpath ) ) continue # check files stats stats = self.get_git_object_default_stats(ref, path) current_stat = os.lstat(full_path) if stats['st_mode'] != current_stat.st_mode: try: os.chmod(full_path, current_stat.st_mode) except OSError: log.info("Repository: Checkout couldn't chmod %s", full_path) self._repo.index.add(self._sanitize(path)) return result def _sanitize(self, path): if path is not None and path.startswith("/"): path = path[1:] return path def push(self, upstream, branch, credentials): """ Push changes from a branch to a remote Examples:: repo.push("origin", "master") """ remote = self.get_remote(upstream) remote.push(["refs/heads/%s" % (branch)], callbacks=credentials) def fetch(self, upstream, branch_name, credentials): """ Fetch from remote and return True if we are behind or False otherwise """ remote = self.get_remote(upstream) remote.fetch(callbacks=credentials) _, behind = self.diverge(upstream, branch_name) self.behind = behind return behind def commit(self, message, author, commiter, parents=None, ref="HEAD"): """ Wrapper for create_commit. It creates a commit from a given ref (default is HEAD) """ status = self._repo.status() if status == {}: return None # sign the author author = Signature(author[0], author[1]) commiter = Signature(commiter[0], commiter[1]) # write index localy tree = self._repo.index.write_tree() self._repo.index.write() # get parent if parents is None: parents = [self._repo.revparse_single(ref).id] return self._repo.create_commit(ref, author, commiter, message, tree, parents) @classmethod def clone(cls, remote_url, path, branch=None, credentials=None): """Clone a repo in a give path and update the working directory with a checkout to head (GIT_CHECKOUT_SAFE_CREATE) :param str remote_url: URL of the repository to clone :param str path: Local path to clone into :param str branch: Branch to checkout after the clone. The default is to use the remote's default branch. """ hasExistingRepo = False try: existingRepoPath = discover_repository(path) if existingRepoPath<>"": hasExistingRepo = True except Exception, e: log.debug("[Exception] discover_repository repo not found: %s", str(e)) pass if hasExistingRepo == False: log.debug("clone_repository %s", path) try: repo = clone_repository(remote_url, path, checkout_branch=branch, callbacks=credentials) except Exception, e: log.error("[Exception] clone_repository failed: %s", str(e)) sys.exit() repo.checkout_head() log.info("repo cloned")