def open_for_write(self, path, flags): global writers fh = self.open_for_read(path, flags) writers += 1 log.debug("CurrentView: Open %s for write", path) return fh
def register(self, routes): for regex, view in routes: log.debug('Registering %s for %s', view, regex) self.routes.append({ 'regex': regex, 'view': view })
def on_idle(self): """ On idle, we have 4 cases: 1. We have to commit and also need to merge some commits from remote. In this case, we commit and announce ourself for merging 2. We are behind from remote, so we announce for merging 3. We only need to commit 4. We announced for merging and nobody is writing in this momement. In this case we are safe to merge and push. """ if not syncing.is_set(): log.debug("Set syncing event (%d pending writes)", writers.value) syncing.set() else: log.debug("Idling (%d pending writes)", writers.value) if writers.value == 0: if self.commits: log.info("Get some commits") self.commit(self.commits) self.commits = [] count = 0 log.debug("Start syncing") while not self.sync(): if count < 5: count += 1 fuzz = random.randint(0, 1000) / 1000 wait = 2 ** count + fuzz log.debug("Failed. Going to sleep for %d seconds", wait) time.sleep(wait) log.debug("Retry-ing")
def __call__(self, operation, *args): """ Magic method which calls a specific method from a view. In Fuse API, almost each method receives a path argument. Based on that path we can route each call to a specific view. For example, if a method which has a path argument like `/current/dir1/dir2/file1` is called, we need to get the certain view that will know how to handle this path, instantiate it and then call our method on the newly created object. :param str operation: Method name to be called :param args: tuple containing the arguments to be transmitted to the method :rtype: function """ if operation in ['destroy', 'init']: view = self else: path = args[0] view, relative_path = self.get_view(path) args = (relative_path,) + args[1:] log.debug('Call %s %s with %r' % (operation, view.__class__.__name__, args)) if not hasattr(view, operation): log.debug('No attribute %s on %s' % (operation, view.__class__.__name__)) raise FuseOSError(ENOSYS) idle.clear() return getattr(view, operation)(*args)
def symlink(self, name, target): result = os.symlink(target, self.repo._full_path(name)) message = "Create symlink to {} for {}".format(target, name) self._stage(add=name, message=message) log.debug("CurrentView: Created symlink to %s from %s", name, target) return result
def unlink(self, path): result = super(CurrentView, self).unlink(path) message = 'Deleted {}'.format(path) self._stage(remove=path, message=message) log.debug("CurrentView: Deleted %s", path) return result
def create(self, path, mode, fi=None): fh = self.open_for_write(path, os.O_WRONLY | os.O_CREAT) super(CurrentView, self).chmod(path, mode) self.dirty[fh] = {'message': "Created {}".format(path), 'stage': True} log.debug("CurrentView: Created %s", path) return fh
def open(self, path, flags): write_mode = flags & (os.O_WRONLY | os.O_RDWR | os.O_APPEND | os.O_CREAT) if write_mode: log.debug("[ignore] open.write_mode: %s", path) return self.open_for_write(path, flags) log.debug("[ignore] open.read_mode: %s", path) return self.open_for_read(path, flags)
def getattr(self, path, fh=None): full_path = self.repo._full_path(path) status = os.lstat(full_path) attrs = dict((key, getattr(status, key)) for key in STATS) attrs.update({"st_uid": self.uid, "st_gid": self.gid}) log.debug("CurrentView: Get attributes %s for %s", str(attrs), path) return attrs
def clean_up(self, local_branch): self.repository.checkout("refs/heads/%s" % local_branch, strategy=pygit2.GIT_CHECKOUT_FORCE) refs = [(target, "refs/heads/" + target) for target in ["merging_local", "merging_remote"]] for branch, ref in refs: log.debug("AcceptMine: Delete %s" % branch) self.repository.lookup_reference(ref).delete()
def create(self, path, mode, fi=None): fh = self.open_for_write(path, os.O_WRONLY | os.O_CREAT) super(CurrentView, self).chmod(path, mode) self.dirty[fh] = { 'message': "Created %s" % path, } log.debug("CurrentView: Created %s", path) return fh
def _parse_ignore_file(self, ignore_file): items = [] log.debug("[ignore] _parse_ignore_file: %s", ignore_file) if ignore_file and os.path.exists(ignore_file): with open(ignore_file) as gitignore: for item in gitignore.readlines(): item = item.strip() if item and not item.startswith('#'): items.append(item) return items
def open_for_write(self, path, flags): global writers fh = self.open_for_read(path, flags) writers += 1 self.dirty[fh] = { "message": "Opened {} for write".format(path), "stage": False } log.debug("CurrentView: Open %s for write", path) return fh
def destroy(self, path): log.debug('Stopping workers') shutting_down.set() fetch.set() for worker in self.workers: worker.join() log.debug('Workers stopped') #shutil.rmtree(self.repo_path) log.info('Successfully umounted %s', self.mount_path)
def link(self, name, target): if target.startswith('/current/'): target = target.replace('/current/', '/') result = super(CurrentView, self).link(target, name) message = "Create link to {} for {}".format(target, name) self._stage(add=name, message=message) log.debug("CurrentView: Created link to %s from %s", name, target) return result
def link(self, name, target): if target.startswith("/%s/" % self.current_path): target = target.replace("/%s/" % self.current_path, "/") result = super(CurrentView, self).link(target, name) message = "Create link to {} for {}".format(target, name) self._stage(add=name, message=message) log.debug("CurrentView: Created link to %s from %s", name, target) return result
def open_for_write(self, path, flags): global writers fh = self.open_for_read(path, flags) writers += 1 self.dirty[fh] = { 'message': "Opened {} for write".format(path), 'stage': False } log.debug("CurrentView: Open %s for write", path) return fh
def destroy(self, path): log.debug('Stopping workers') shutting_down.set() fetch.set() for worker in self.workers: worker.join() log.debug('Workers stopped') shutil.rmtree(self.repo_path) log.info('Successfully umounted %s', self.mount_path)
def link(self, name, target): if target.startswith('/current/'): target = target.replace('/current/', '/') result = super(CurrentView, self).link(target, name) message = "Create link to %s for %s" % (target, name) self._stage(add=name, message=message) log.debug("CurrentView: Created link to %s from %s", name, target) return result
def fsync(self, path, fdatasync, fh): """ Each time you fsync, a new commit and push are made """ result = super(CurrentView, self).fsync(path, fdatasync, fh) message = "Fsync {}".format(path) self._stage(add=path, message=message) log.debug("CurrentView: Fsync %s", path) return result
def unlink(self, path, gitignore): result = super(CurrentView, self).unlink(path) if gitignore == True: log.debug("[ignore] unlink:%s", path) return result message = 'Deleted {}'.format(path) self._stage(remove=path, message=message) log.debug("CurrentView: Deleted %s", path) return result
def fetch(self): with remote_operation: fetch.clear() try: log.debug("Start fetching") self.repository.fetch(self.upstream, self.branch) fetch_successful.set() log.debug("Fetch done") except: fetch_successful.clear() log.exception("Fetch failed")
def getattr(self, path, fh=None): full_path = self.repo._full_path(path) status = os.lstat(full_path) attrs = dict((key, getattr(status, key)) for key in STATS) attrs.update({ 'st_uid': self.uid, 'st_gid': self.gid, }) log.debug("CurrentView: Get attributes %s for %s", str(attrs), path) return attrs
def fsync(self, path, fdatasync, fh): """ Each time you fsync, a new commit and push are made """ result = super(CurrentView, self).fsync(path, fdatasync, fh) message = 'Fsync {}'.format(path) self._stage(add=path, message=message) log.debug("CurrentView: Fsync %s", path) return result
def rename(self, old, new): new = re.sub(self.regex, '', new) result = super(CurrentView, self).rename(old, new) message = "Rename %s to %s" % (old, new) self._stage(**{ 'remove': os.path.split(old)[1], 'add': new, 'message': message }) log.debug("CurrentView: Renamed %s to %s", old, new) return result
def rename(self, old, new): new = re.sub(self.regex, '', new) result = super(CurrentView, self).rename(old, new) message = "Rename {} to {}".format(old, new) self._stage(**{ 'remove': os.path.split(old)[1], 'add': new, 'message': message }) log.debug("CurrentView: Renamed %s to %s", old, new) return result
def rename(self, old, new): new = re.sub(self.regex, "", new) result = super(CurrentView, self).rename(old, new) message = "Rename {} to {}".format(old, new) self._stage(**{ "remove": os.path.split(old)[1], "add": new, "message": message }) log.debug("CurrentView: Renamed %s to %s", old, new) return result
def work(self): while True: timeout = self.timeout if idle.is_set(): timeout = self.idle_timeout log.debug("Wait for {}".format(timeout)) fetch.wait(timeout) if shutting_down.is_set(): log.info("Stop fetch worker") break self.fetch()
def run(self): while True: if shutting_down.is_set(): log.info("Stop sync worker") break try: job = self.commit_queue.get(timeout=self.timeout, block=True) if job['type'] == 'commit': self.commits.append(job) log.debug("Got a commit job") except Empty: log.debug("Nothing to do right now, going idle") self.on_idle()
def work(self): while True: if shutting_down.is_set(): log.info("Stop sync worker") break try: job = self.commit_queue.get(timeout=self.timeout, block=True) if job['type'] == 'commit': self.commits.append(job) log.debug("Got a commit job") except Empty: log.debug("Nothing to do right now, going idle") self.on_idle()
def link(self, name, target, gitignore): if target.startswith('/%s/' % self.current_path): target = target.replace('/%s/' % self.current_path, '/') result = super(CurrentView, self).link(target, name) if gitignore == True: log.debug("[ignore] link:%s", name) return result message = "Create link to {} for {}".format(target, name) self._stage(add=name, message=message) log.debug("CurrentView: Created link to %s from %s", name, target) return result
def fetch(self): with remote_operation: fetch.clear() try: log.debug("Start fetching") was_behind = self.repository.fetch(self.upstream, self.branch) fetch_successful.set() if was_behind: log.info("Fetch done") else: log.debug("Nothing to fetch") except: fetch_successful.clear() log.exception("Fetch failed")
def write(self, path, buf, offset, fh): """ We don't like big big files, so we need to be really carefull with them. First we check for offset, then for size. If any of this is off limit, raise EFBIG error and delete the file. """ if offset + len(buf) > self.max_size: raise FuseOSError(errno.EFBIG) result = super(CurrentView, self).write(path, buf, offset, fh) self.dirty[fh] = {"message": "Update {}".format(path), "stage": True} log.debug("CurrentView: Wrote %s to %s", len(buf), path) return result
def release(self, path, fh): """ Check for path if something was written to. If so, commit and push the changed to upstream. """ if fh in self.dirty: message = self.dirty[fh]['message'] del self.dirty[fh] global writers writers -= 1 log.debug("CurrentView: Staged %s for commit", path) self._stage(add=path, message=message) log.debug("CurrentView: Release %s", path) return os.close(fh)
def fsync(self, path, fdatasync, fh, gitignore): """ Each time you fsync, a new commit and push are made """ result = super(CurrentView, self).fsync(path, fdatasync, fh) if gitignore == True: log.debug("[ignore] fsync:%s", path) return result message = 'Fsync {}'.format(path) self._stage(add=path, message=message) log.debug("CurrentView: Fsync %s", path) return result
def decorated(*args, **kwargs): if not fetch_successful.is_set() or not push_successful.is_set(): raise FuseOSError(EROFS) global writers writers += 1 if syncing.is_set(): log.debug("WriteOperation: Wait until syncing is done") sync_done.wait() try: result = f(*args, **kwargs) finally: writers -= 1 return result
def commit(self, add=None, message=None, remove=None): if message is None: raise ValueError("Message shoduld not be None") if add is None and remove is None: message = "You need to add or to remove some files from/to index" raise ValueError(message) self.queue.put({ 'type': 'commit', 'params': { 'add': self._to_list(add), 'message': message, 'remove': self._to_list(remove), } }) log.debug("Got a new commit job on queue")
def rmdir(self, path): message = 'Delete the %s directory' % path # Unlink all the files full_path = self.repo._full_path(path) for root, dirs, files in os.walk(full_path): for _file in files: deleting_file = os.path.join(root, _file) if os.path.exists(deleting_file): result = super(CurrentView, self).unlink(os.path.join(path, _file)) self._stage(remove=os.path.join(path, _file), message=message) # Delete the actual directory result = super(CurrentView, self).rmdir("%s/" % path) log.debug("CurrentView: %s", message) return result
def rename(self, old, new, gitignore): new = re.sub(self.regex, '', new) result = super(CurrentView, self).rename(old, new) if gitignore == True: log.debug("[ignore] rename:%s", name) return result message = "Rename {} to {}".format(old, new) self._stage(**{ 'remove': os.path.split(old)[1], 'add': new, 'message': message }) log.debug("CurrentView: Renamed %s to %s", old, new) return result
def rmdir(self, path): message = 'Delete the {} directory'.format(path) # Unlink all the files full_path = self.repo._full_path(path) for root, dirs, files in os.walk(full_path): for _file in files: deleting_file = os.path.join(root, _file) if os.path.exists(deleting_file): result = super(CurrentView, self).unlink(os.path.join(path, _file)) self._stage(remove=os.path.join(path, _file), message=message) # Delete the actual directory result = super(CurrentView, self).rmdir("{}/".format(path)) log.debug("CurrentView: %s", message) return result
def open_for_write(self, path, flags, gitignore): fh = self.open_for_read(path, flags) if gitignore == True: log.debug("[ignore] open_for_write:%s", path) return fh global writers writers += 1 self.dirty[fh] = { 'message': "Opened {} for write".format(path), 'stage': False } log.debug("CurrentView: Open %s for write", path) return fh
def write(self, path, buf, offset, fh): """ We don't like big big files, so we need to be really carefull with them. First we check for offset, then for size. If any of this is off limit, raise EFBIG error and delete the file. """ if offset + len(buf) > self.max_size: raise FuseOSError(errno.EFBIG) result = super(CurrentView, self).write(path, buf, offset, fh) self.dirty[fh] = { 'message': 'Update %s' % path, } log.debug("CurrentView: Wrote %s to %s", len(buf), path) return result
def chmod(self, path, mode): """ Executes chmod on the file at os level and then it commits the change. """ str_mode = ('%o' % mode)[-4:] if str_mode not in ['0755', '0644']: raise FuseOSError(errno.EINVAL) result = super(CurrentView, self).chmod(path, mode) if os.path.isdir(self.repo._full_path(path)): return result message = 'Chmod to {} on {}'.format(str_mode, path) self._stage(add=path, message=message) log.debug("CurrentView: Change %s mode to %s", path, ('0%o' % mode)[-4:]) return result
def chmod(self, path, mode): """ Executes chmod on the file at os level and then it commits the change. """ str_mode = ("%o" % mode)[-4:] if str_mode not in ["0755", "0644"]: raise FuseOSError(errno.EINVAL) result = super(CurrentView, self).chmod(path, mode) if os.path.isdir(self.repo._full_path(path)): return result message = "Chmod to {} on {}".format(str_mode, path) self._stage(add=path, message=message) log.debug("CurrentView: Change %s mode to %s", path, ("0%o" % mode)[-4:]) return result
def chmod(self, path, mode): """ Executes chmod on the file at os level and then it commits the change. """ str_mode = ('%o' % mode)[-4:] if str_mode not in ['0755', '0644']: raise FuseOSError(errno.EINVAL) result = super(CurrentView, self).chmod(path, mode) if os.path.isdir(self.repo._full_path(path)): return result message = 'Chmod to %s on %s' % (str_mode, path) self._stage(add=path, message=message) log.debug("CurrentView: Change %s mode to %s", path, ('0%o' % mode)[-4:]) return result
def commit(self, jobs): if len(jobs) == 1: message = jobs[0]['params']['message'] else: updates = set([]) number_of_removal = 0 number_of_additions = 0 for job in jobs: removal_set = set(job['params']['remove']) addition_set = set(job['params']['add']) number_of_removal += len(removal_set) number_of_additions += len(addition_set) updates = updates | removal_set | addition_set message = "Update {} items. ".format(len(updates)) if number_of_additions: message += "Added {} items. ".format(number_of_additions) if number_of_removal: message += "Removed {} items. ".format(number_of_removal) message = message.strip() old_head = self.repository.head.target new_commit = self.repository.commit(message, self.author, self.commiter) if new_commit: log.debug("Commit %s with %s as author and %s as commiter", message, self.author, self.commiter) self.repository.commits.update() log.debug("Update commits cache") else: self.repository.create_reference("refs/heads/%s" % self.branch, old_head, force=True) self.repository.checkout_head(strategy=pygit2.GIT_CHECKOUT_FORCE) log.debug("Checkout to HEAD")
def commit(self, jobs): if len(jobs) == 1: message = jobs[0]['params']['message'] else: updates = set([]) for job in jobs: updates = updates | set(job['params']['add']) updates = updates | set(job['params']['remove']) message = "Update {} items".format(len(updates)) old_head = self.repository.head.target new_commit = self.repository.commit(message, self.author, self.commiter) if new_commit: log.debug("Commit %s with %s as author and %s as commiter", message, self.author, self.commiter) self.repository.commits.update() log.debug("Update commits cache") else: self.repository.create_reference("refs/heads/%s" % self.branch, old_head, force=True) self.repository.checkout_head(strategy=pygit2.GIT_CHECKOUT_FORCE) log.debug("Checkout to HEAD")
def mkdir(self, path, mode): result = super(CurrentView, self).mkdir(path, mode) keep_path = "%s/.keep" % path full_path = self.repo._full_path(keep_path) if not os.path.exists(keep_path): global writers fh = os.open(full_path, os.O_WRONLY | os.O_CREAT) writers += 1 log.info("CurrentView: Open %s for write", full_path) super(CurrentView, self).chmod(keep_path, 0o644) self.dirty[fh] = { 'message': "Create the %s directory" % path, } self.release(keep_path, fh) log.debug("CurrentView: Created directory %s with mode %s", path, mode) return result
def work(self): idle_times = 0 while True: if shutting_down.is_set(): log.info("Stop sync worker") break try: job = self.commit_queue.get(timeout=self.timeout, block=True) if job['type'] == 'commit': self.commits.append(job) log.debug("Got a commit job") idle_times = 0 idle.clear() except Empty: log.debug("Nothing to do right now, going idle") if idle_times > self.min_idle_times: idle.set() idle_times += 1 self.on_idle()
def merge(self): log.debug("Start merging") self.strategy(self.branch, self.branch, self.upstream) log.debug("Update commits cache") self.repository.commits.update() log.debug("Update ignore list") self.repository.ignore.update()
def get_view(self, path): """ Try to map a given path to it's specific view. If a match is found, a view object is created with the right regex groups(named or unnamed). :param str path: path to be matched :rtype: view object, relative path """ for route in self.routes: result = re.search(route['regex'], path) if result is None: continue groups = result.groups() relative_path = re.sub(route['regex'], '', path) relative_path = '/' if not relative_path else relative_path cache_key = result.group(0) log.debug("Router: Cache key for %s: %s", path, cache_key) view = lru_cache.get_if_exists(cache_key) if view is not None: log.debug("Router: Serving %s from cache", path) return view, relative_path kwargs = result.groupdict() # TODO: move all this to a nice config variable kwargs['repo'] = self.repo kwargs['ignore'] = self.repo.ignore kwargs['repo_path'] = self.repo_path kwargs['mount_path'] = self.mount_path kwargs['regex'] = route['regex'] kwargs['relative_path'] = relative_path kwargs['current_path'] = self.current_path kwargs['history_path'] = self.history_path kwargs['uid'] = self.uid kwargs['gid'] = self.gid kwargs['branch'] = self.branch kwargs['mount_time'] = self.mount_time kwargs['queue'] = self.commit_queue kwargs['max_size'] = self.max_size kwargs['max_offset'] = self.max_offset args = set(groups) - set(kwargs.values()) view = route['view'](*args, **kwargs) lru_cache[cache_key] = view log.debug("Router: Added %s to cache", path) return view, relative_path raise ValueError("Found no view for '{}'".format(path))