def __init__(self, origin, branch="master", path="."): self.origin = origin self.branch = branch self.root = os.path.realpath(path) self.halt = False self.rwlock = Lock() # Can't use the default rlock here since we want to acquire/release from different threads self.sync_c = Condition(Lock()) self.timer_c = Condition(Lock()) self.timer = None self.handlers = { "ping": self._handlePing, "lock": self._handleLock, "unlock": self._handleUnlock, "info": self._handleInfo, } self.lock_timer = None self.lock_lock = Condition() self.locks = {} self.lock_expire_time = time() self.control_dir = self.getControlDirectory(self.root) try: os.mkdir(self.control_dir) except OSError: pass self.control_socket_path = self.getControlSocketPath(self.root) client = GitFSClient(self.root) self.needSyncTime = None try: if client.pingRemote(): # There is another file system mounted. logging.debug("Exiting because file system already mounted.\n") raise FuseOSError(EBUSY) except socket.error as se: logging.debug("socket.error = %s" % se) pass client.close() client = None try: os.remove(self.control_socket_path) except OSError: pass self.control_server = None self.control_server = ThreadingUnixStreamServer( self.control_socket_path, type( "GitFSRequestHandler", (PacketizeMixIn, BaseRequestHandler, object), dict( fs=self, dictFromString=self.parseDict, stringFromDict=self.marshalDict, handleDict=lambda s, d: s.fs._handleRequest(s, d), ), ), ) self.control_server.daemon_threads = True # setup the threads last so that they don't prevent an exit. self.control_thread = Thread(target=self.control_server.serve_forever, args=()) self.control_thread.start() self.repo = GitRepo(path, origin, branch, sync=True) self.sync_thread = Thread(target=self._sync, args=()) self.sync_thread.start()
def start_server(): server = ThreadingUnixStreamServer(unix_socket_path(), RequestHandler) server.serve_forever()
class GitFS(GitFSStringMixIn, Operations): """A simple filesystem using Git and FUSE. """ def __init__(self, origin, branch="master", path="."): self.origin = origin self.branch = branch self.root = os.path.realpath(path) self.halt = False self.rwlock = Lock() # Can't use the default rlock here since we want to acquire/release from different threads self.sync_c = Condition(Lock()) self.timer_c = Condition(Lock()) self.timer = None self.handlers = { "ping": self._handlePing, "lock": self._handleLock, "unlock": self._handleUnlock, "info": self._handleInfo, } self.lock_timer = None self.lock_lock = Condition() self.locks = {} self.lock_expire_time = time() self.control_dir = self.getControlDirectory(self.root) try: os.mkdir(self.control_dir) except OSError: pass self.control_socket_path = self.getControlSocketPath(self.root) client = GitFSClient(self.root) self.needSyncTime = None try: if client.pingRemote(): # There is another file system mounted. logging.debug("Exiting because file system already mounted.\n") raise FuseOSError(EBUSY) except socket.error as se: logging.debug("socket.error = %s" % se) pass client.close() client = None try: os.remove(self.control_socket_path) except OSError: pass self.control_server = None self.control_server = ThreadingUnixStreamServer( self.control_socket_path, type( "GitFSRequestHandler", (PacketizeMixIn, BaseRequestHandler, object), dict( fs=self, dictFromString=self.parseDict, stringFromDict=self.marshalDict, handleDict=lambda s, d: s.fs._handleRequest(s, d), ), ), ) self.control_server.daemon_threads = True # setup the threads last so that they don't prevent an exit. self.control_thread = Thread(target=self.control_server.serve_forever, args=()) self.control_thread.start() self.repo = GitRepo(path, origin, branch, sync=True) self.sync_thread = Thread(target=self._sync, args=()) self.sync_thread.start() def _lockWithTimeOut(self, name, t): if t <= 0: return self.lock_lock.acquire() expt = t + time() self.locks[name] = expt if self.lock_expire_time - expt < 0: self.lock_expire_time = expt if self.lock_timer != None: self.lock_timer.cancel() else: logging.debug("Aquiring fresh lock") self.sync_c.acquire() self.lock_timer = Timer(t, self._lockTimerExpire, args=()) self.lock_timer.start() self.lock_lock.release() def _lockTimerExpire(self): logging.debug("_lockTimeExpire") self.lock_lock.acquire() self.__lockTimerExpire() self.lock_lock.release() def __lockTimerExpire(self): logging.debug("__lockTimeExpire") now = time() t = now for key in self.locks: if t - self.locks[key] < 0: t = self.locks[key] if now - self.locks[key] > 300: del self.locks[key] if t - now > 0: self.lock_expire_time = t if self.lock_timer != None: self.lock_timer.cancel() else: logging.debug("***** ERROR ***** __lockTimerExpire doesn't have lock. acquiring") self.sync_c.acquire() self.lock_timer = Timer(t - now, self._lockTimerExpire, args=()) self.lock_timer.start() logging.debug("extending lock.") else: if self.lock_timer != None: logging.debug("releasing lock.") self.lock_timer.cancel() self.lock_timer = None self.sync_c.release() def _unlock(self, name): if name not in self.locks: return self.lock_lock.acquire() t = self.locks[name] del self.locks[name] if t >= self.lock_expire_time or len(keys(self.locks)) == 0: self.__lockTimerExpire() self.lock_lock.release() def _handleRequest(self, request, d): if d["action"] in self.handlers: mf = self.handlers[d["action"]] return mf(d, request) logging.debug("No request in packet: %s" % d) self._respond(request, {"status": "Unknown Command"}) return None def _respond(self, request, responseDict): request.sendDict(responseDict) def _handlePing(self, reqDict, request): self._respond(request, {"status": "ok", "message": "pong"}) def _handleLock(self, reqDict, request): self._lockWithTimeOut("%s" % request.request.fileno(), 60) self._respond(request, {"status": "ok", "name": "%s" % request.request.fileno()}) def _handleUnlock(self, reqDict, request): self._unlock("%s" % request.request.fileno()) self._respond(request, {"status": "ok"}) def _handleInfo(self, reqDict, request): self._respond(request, {"status": "ok", "origin": self.repo.origin, "branch": self.repo.branch}) def _sync(self): while True: self.sync_c.acquire() if not self.halt: # wait till a sync request comes self.sync_c.wait() self.timer_c.acquire() if self.timer != None: self.timer.cancel() self.timer = None self.timer_c.release() try: if not self.repo.synchronize(): # sync failed, we need to try again. self.needSync() self.needSyncTime = None except Exception as e: logging.debug("synchronize threw exception %s" % e) self.sync_c.release() # can't release this until sync is complete because we can't change files while we sync. else: self.repo.forcePush() self.repo.push() self.sync_c.release() break def forceSync(self): logging.debug("forceSync()") self.sync_c.acquire() self.sync_c.notifyAll() self.sync_c.release() def needSync(self): logging.debug("needSync()") self.timer_c.acquire() if self.needSyncTime == None: self.needSyncTime = time() if self.timer != None: if time() - self.needSyncTime > 5 * 60: return self.timer.cancel() # don't do anything until there is a pause. self.timer = Timer(10, self.forceSync, args=()) self.timer.start() self.timer_c.release() def shutdown(self): # stop sync thread self.sync_c.acquire() self.halt = True self.sync_c.notifyAll() self.sync_c.release() self.repo.shutDown() def destroy(self, path): self.shutdown() if self.control_server != None: self.control_server.shutdown() try: os.remove(self.control_socket_path) except OSError: pass def __call__(self, op, path, *args): try: logging.debug("calling %s on %s" % (op, path)) path = self.escapePath(path) r = super(GitFS, self).__call__(op, self.root + path, *args) pr = "%s" % r pr = pr[:10] logging.debug("returning %s for %s" % (pr, op)) return r except Exception as e: logging.debug("Unhandled exception %s" % e) raise e def access(self, path, mode): if not os.access(path, mode): raise FuseOSError(EACCES) def chmod(self, path, mode): self.needSync() return os.chmod(path, mode) def chown(self, path, uid, gid): self.needSync() return super(GitFS, self).chown(path, uid, gid) def create(self, path, mode): # XXXXX Fixme what should the flags be? return os.open(path, os.O_RDWR | os.O_CREAT, mode) def flush(self, path, fh): self.sync_c.acquire() self.sync_c.notifyAll() self.sync_c.release() return os.fsync(fh) def fsync(self, path, datasync, fh): self.needSync() return os.fsync(fh) def fsyncdir(self, path, datasync, fh): return self.fsync(path, datasync, fh) def getattr(self, path, fh=None): st = os.lstat(path) return dict( (key, getattr(st, key)) for key in ("st_atime", "st_ctime", "st_gid", "st_mode", "st_mtime", "st_nlink", "st_size", "st_uid") ) getxattr = None def link(self, target, source): source = self.escapePath(source) self.needSync() return os.link(source, target) listxattr = None mknod = os.mknod def mkdir(self, path, mode): return os.mkdir(path, mode) def open(self, path, fip): f = os.open(path, fip) logging.debug("open(%s, %s): %d" % (path, fip, f)) return f def read(self, path, size, offset, fh): with self.rwlock: os.lseek(fh, offset, 0) return os.read(fh, size) def readdir(self, path, fh): files = os.listdir(path) uefiles = [] for file in files: if self.isValidPath(file): uefiles = uefiles + [self.unescapePath(file)] return [".", ".."] + uefiles readlink = os.readlink def release(self, path, fh): return os.close(fh) def rename(self, old, new): self.needSync() return os.rename(old, self.root + self.escapePath(new)) def rmdir(self, path): self.needSync() return os.rmdir(path) def statfs(self, path): stv = os.statvfs(path) return dict( (key, getattr(stv, key)) for key in ( "f_bavail", "f_bfree", "f_blocks", "f_bsize", "f_favail", "f_ffree", "f_files", "f_flag", "f_frsize", "f_namemax", ) ) def symlink(self, target, source): self.needSync() return os.symlink(source, target) def truncate(self, path, length, fh=None): self.needSync() with open(path, "r+") as f: f.truncate(length) def unlink(self, path): self.needSync() return os.unlink(path) utimens = os.utime def write(self, path, data, offset, fh): self.needSync() with self.rwlock: os.lseek(fh, offset, 0) return os.write(fh, data)
def __init__(self, origin, branch='master', path='.', mount_point='.'): super(GitFS, self).__init__() self.origin = origin self.branch = branch self.root = os.path.realpath(path) self.mount_point = mount_point self.halt = False self.rwlock = Lock() self.need_sync_time = None # Can't use the default rlock here since we want to acquire/release from different threads self.sync_c = Condition(Lock()) self.timer_c = Condition(Lock()) self.id = None self.timer = None self.handlers = { 'ping': self._handlePing, 'lock': self._handleLock, 'unlock': self._handleUnlock, 'info': self._handleInfo, 'getConfig': self._getConfig } self.lock_timer = None self.lock_lock = Condition() self.locks = {} self.lock_expire_time = time() self.control_dir = self.getControlDirectory() try: os.makedirs(self.control_dir) except OSError: pass self.info_dir = self.getInfoDirectory(self.root) try: os.makedirs(self.info_dir) except OSError: pass self.control_socket_path = self.getControlSocketPath(self.getID(), server=True) self.lockGitFSDir() try: try: client = GitFSClient.getClientByPath(self.mount_point, False, False) raise FuseOSError(EBUSY) except GitFSError: pass try: os.remove(self.control_socket_path) except OSError: pass self.control_server = None self.control_server = ThreadingUnixStreamServer( self.control_socket_path, type( "GitFSRequestHandler", (PacketizeMixIn, BaseRequestHandler, object), dict(fs=self, dictFromString=self.parseDict, stringFromDict=self.marshalDict, handleDict=lambda s, d: s.fs._handleRequest(s, d)))) self.control_server.daemon_threads = True # setup the threads last so that they don't prevent an exit. self.control_thread = Thread( target=self.control_server.serve_forever, args=()) self.control_thread.start() finally: self.unlockGitFSDir() mt = self.getMTab() mt[mount_point] = self.getID() self.updateMTab(mt) self.repo = GitRepo(path, origin, branch, sync=True) self.sync_thread = Thread(target=self._sync, args=()) self.sync_thread.start()
# -*- coding: utf-8 -*- """ 使用python标准库的 ThreadingTCPServer : 如果想用gevent, 可以用gevent patch all """ from SocketServer import ThreadingUnixStreamServer, StreamRequestHandler from mprpc import RPCHandler class SumHandler(RPCHandler): def sum(self, x, y): return x + y class _Handler(StreamRequestHandler): def handle(self): SumHandler()(self.connection, None) server = ThreadingUnixStreamServer('./x.sock', _Handler) server.serve_forever()
sys.exit() sys.stdout.flush() sys.stderr.flush() si = file('/dev/null', 'r') so = file('/dev/null', 'a+') se = file('/dev/null', 'a+', 0) os.dup2(si.fileno(), sys.stdin.fileno()) os.dup2(so.fileno(), sys.stdout.fileno()) os.dup2(se.fileno(), sys.stderr.fileno()) if __name__ == '__main__': logger = configure_logging() # Socket server creates its own socket file. Delete if it exists already. if os.path.exists(SOCKET_FILENAME): logger.warning('Unlinking existing socket: %s' % SOCKET_FILENAME) os.unlink(SOCKET_FILENAME) server = ThreadingUnixStreamServer(SOCKET_FILENAME, GitHubPingHandler) os.chmod(SOCKET_FILENAME, 0777) try: logger.info('Starting Gitte socket server at: %s' % SOCKET_FILENAME) make_service() server.serve_forever() except KeyboardInterrupt: os.unlink(SOCKET_FILENAME) print "\nKeyboard interupt recieved, Gitte server stopping..."
class GitFS(GitFSBase, Operations): """A simple filesystem using Git and FUSE. """ def __init__(self, origin, branch='master', path='.', mount_point='.'): super(GitFS, self).__init__() self.origin = origin self.branch = branch self.root = os.path.realpath(path) self.mount_point = mount_point self.halt = False self.rwlock = Lock() self.need_sync_time = None # Can't use the default rlock here since we want to acquire/release from different threads self.sync_c = Condition(Lock()) self.timer_c = Condition(Lock()) self.id = None self.timer = None self.handlers = { 'ping': self._handlePing, 'lock': self._handleLock, 'unlock': self._handleUnlock, 'info': self._handleInfo, 'getConfig': self._getConfig } self.lock_timer = None self.lock_lock = Condition() self.locks = {} self.lock_expire_time = time() self.control_dir = self.getControlDirectory() try: os.makedirs(self.control_dir) except OSError: pass self.info_dir = self.getInfoDirectory(self.root) try: os.makedirs(self.info_dir) except OSError: pass self.control_socket_path = self.getControlSocketPath(self.getID(), server=True) self.lockGitFSDir() try: try: client = GitFSClient.getClientByPath(self.mount_point, False, False) raise FuseOSError(EBUSY) except GitFSError: pass try: os.remove(self.control_socket_path) except OSError: pass self.control_server = None self.control_server = ThreadingUnixStreamServer( self.control_socket_path, type( "GitFSRequestHandler", (PacketizeMixIn, BaseRequestHandler, object), dict(fs=self, dictFromString=self.parseDict, stringFromDict=self.marshalDict, handleDict=lambda s, d: s.fs._handleRequest(s, d)))) self.control_server.daemon_threads = True # setup the threads last so that they don't prevent an exit. self.control_thread = Thread( target=self.control_server.serve_forever, args=()) self.control_thread.start() finally: self.unlockGitFSDir() mt = self.getMTab() mt[mount_point] = self.getID() self.updateMTab(mt) self.repo = GitRepo(path, origin, branch, sync=True) self.sync_thread = Thread(target=self._sync, args=()) self.sync_thread.start() def getID(self): if self.id is None: self.id = mUUID.getUUIDFromFile(self.getUUIDFile(self.root), create=True).toString() return self.id def _lockWithTimeOut(self, name, t): if t <= 0: return self.lock_lock.acquire() expt = t + time() self.locks[name] = expt if self.lock_expire_time - expt < 0: self.lock_expire_time = expt if self.lock_timer != None: self.lock_timer.cancel() else: logging.debug("Aquiring fresh lock") self.sync_c.acquire() self.lock_timer = Timer(t, self._lockTimerExpire, args=()) self.lock_timer.start() self.lock_lock.release() def _lockTimerExpire(self): logging.debug('_lockTimeExpire') self.lock_lock.acquire() self.__lockTimerExpire() self.lock_lock.release() def __lockTimerExpire(self): logging.debug('__lockTimeExpire') now = time() t = now for key in self.locks: if t - self.locks[key] < 0: t = self.locks[key] if now - self.locks[key] > 300: del self.locks[key] if t - now > 0: self.lock_expire_time = t if self.lock_timer != None: self.lock_timer.cancel() else: logging.debug( "***** ERROR ***** __lockTimerExpire doesn't have lock. acquiring" ) self.sync_c.acquire() self.lock_timer = Timer(t - now, self._lockTimerExpire, args=()) self.lock_timer.start() logging.debug("extending lock.") else: if self.lock_timer != None: logging.debug("releasing lock.") self.lock_timer.cancel() self.lock_timer = None self.sync_c.release() def _unlock(self, name): if name not in self.locks: return self.lock_lock.acquire() t = self.locks[name] del self.locks[name] if t >= self.lock_expire_time or len(keys(self.locks)) == 0: self.__lockTimerExpire() self.lock_lock.release() def _handleRequest(self, request, d): if d['action'] in self.handlers: mf = self.handlers[d['action']] return mf(d, request) logging.debug("No request in packet: %s" % d) self._respond(request, {'status': 'Unknown Command'}) return None def _respond(self, request, responseDict): request.sendDict(responseDict) def _handlePing(self, reqDict, request): self._respond(request, {'status': 'ok', 'message': 'pong'}) def _handleLock(self, reqDict, request): self._lockWithTimeOut('%s' % request.request.fileno(), 60) self._respond(request, { 'status': 'ok', 'name': '%s' % request.request.fileno() }) def _handleUnlock(self, reqDict, request): self._unlock('%s' % request.request.fileno()) self._respond(request, {'status': 'ok'}) def _handleInfo(self, reqDict, request): self._respond( request, { 'status': 'ok', 'origin': self.repo.origin, 'branch': self.repo.branch, 'root': self.root, 'path': self.mount_point }) def _getConfig(self, reqDict, request): key = reqDict['key'] resp = self.getConfigForInstance(key) self._respond(request, {'status': 'ok', key: resp}) def _sync(self): while True: self.sync_c.acquire() if not self.halt: # wait till a sync request comes self.sync_c.wait() self.timer_c.acquire() if self.timer != None: self.timer.cancel() self.timer = None self.timer_c.release() try: if not self.repo.synchronize(): #sync failed, we need to try again. self.needSync() self.need_sync_time = None except Exception as e: logging.debug("synchronize threw exception %s" % e) self.sync_c.release( ) # can't release this until sync is complete because we can't change files while we sync. else: self.repo.forcePush() self.repo.push() self.sync_c.release() break def getHostInfo(): if self.hostinfo is None: self.hostinfo = HostInfo() self.hostinfo.update() return self.hostinfo def matchesInstance(self, key): """ returns a number determinging how well the past in key matches the current instance.""" if key == 'default': return .1 #Very weak match hostinfo = self.getHostInfo() try: ip = socket.inet_pton(key) return hostinfo.matchAddress(ip) except SocketError: pass return hostinfo.matchHostName(key) def getConfigForInstanceSingleFile(self, key, name): logging.debug('getConfigForInstanceSingleFile(%s, %s)' % (key, name)) c = self.getConfig(name) if c is None: return None match = 0 if key not in c: return None v = c[key] logging.debug('getConfigForInstanceSingleFile(%s, %s) value=%s' % (key, name, v)) if isinstance(v, dict): rv = None for (key, value) in v.iteritems(): tm = self.matchesInstance(key) if tm > match: match = tm rv = value v = rv if v is None: return None if isinstance(v, list): v = random.choose(v) return (match, v) def getConfigForInstance(self, key): match = 0 value = None logging.debug('getConfigForInstance(%s)' % key) if key in self.config_file_priorities: filenames = self.config_file_priorities[key] else: filenames = self.config_file_priorities['default'] for name in filenames: a = self.getConfigForInstanceSingleFile(key, name) if a is None: continue (m, v) = a if (m >= match): match = m value = v return value def forceSync(self): logging.debug('forceSync()') self.sync_c.acquire() self.sync_c.notifyAll() self.sync_c.release() def needSync(self): logging.debug('needSync()') self.timer_c.acquire() if self.need_sync_time is None: self.need_sync_time = time() if self.timer != None: if time() - self.need_sync_time > 5 * 60: return self.timer.cancel() # don't do anything until there is a pause. self.timer = Timer(10, self.forceSync, args=()) self.timer.start() self.timer_c.release() def shutdown(self): # stop sync thread self.sync_c.acquire() self.halt = True self.sync_c.notifyAll() self.sync_c.release() self.repo.shutDown() def destroy(self, path): self.shutdown() if self.control_server != None: self.control_server.shutdown() try: os.remove(self.control_socket_path) except OSError: pass def __call__(self, op, path, *args): try: logging.debug("calling %s on %s" % (op, path)) path = self.escapePath(path) r = super(GitFS, self).__call__(op, self.root + path, *args) pr = "%s" % r pr = pr[:10] logging.debug("returning %s for %s" % (pr, op)) return r except Exception as e: logging.debug("Unhandled exception %s" % e) raise e def access(self, path, mode): if not os.access(path, mode): raise FuseOSError(EACCES) def chmod(self, path, mode): self.needSync() return os.chmod(path, mode) def chown(self, path, uid, gid): self.needSync() return super(GitFS, self).chown(path, uid, gid) def create(self, path, mode): # XXXXX Fixme what should the flags be? return os.open(path, os.O_RDWR | os.O_CREAT, mode) def flush(self, path, fh): self.sync_c.acquire() self.sync_c.notifyAll() self.sync_c.release() return os.fsync(fh) def fsync(self, path, datasync, fh): self.needSync() return os.fsync(fh) def fsyncdir(self, path, datasync, fh): return self.fsync(path, datasync, fh) def getattr(self, path, fh=None): st = os.lstat(path) return dict((key, getattr(st, key)) for key in ('st_atime', 'st_ctime', 'st_gid', 'st_mode', 'st_mtime', 'st_nlink', 'st_size', 'st_uid')) getxattr = None def link(self, target, source): source = self.escapePath(source) self.needSync() return os.link(source, target) listxattr = None mknod = os.mknod def mkdir(self, path, mode): return os.mkdir(path, mode) def open(self, path, fip): f = os.open(path, fip) logging.debug("open(%s, %s): %d" % (path, fip, f)) return f def read(self, path, size, offset, fh): with self.rwlock: os.lseek(fh, offset, 0) return os.read(fh, size) def readdir(self, path, fh): files = os.listdir(path) uefiles = [] for file in files: if self.isValidPath(file): uefiles = uefiles + [self.unescapePath(file)] return ['.', '..'] + uefiles readlink = os.readlink def release(self, path, fh): return os.close(fh) def rename(self, old, new): self.needSync() return os.rename(old, self.root + self.escapePath(new)) def rmdir(self, path): self.needSync() return os.rmdir(path) def statfs(self, path): stv = os.statvfs(path) return dict((key, getattr(stv, key)) for key in ('f_bavail', 'f_bfree', 'f_blocks', 'f_bsize', 'f_favail', 'f_ffree', 'f_files', 'f_flag', 'f_frsize', 'f_namemax')) def symlink(self, target, source): self.needSync() return os.symlink(source, target) def truncate(self, path, length, fh=None): self.needSync() with open(path, 'r+') as f: f.truncate(length) def unlink(self, path): self.needSync() return os.unlink(path) utimens = os.utime def write(self, path, data, offset, fh): self.needSync() with self.rwlock: os.lseek(fh, offset, 0) return os.write(fh, data)
def return_unix(sock='spamd.sock'): """Return a unix SPAMD server""" if os.path.exists(sock): os.remove(sock) server = ThreadingUnixStreamServer(sock, TestSpamdHandler) return server
def __init__(self, address, handler, cqueue): self.queue = cqueue ThreadingUnixStreamServer.__init__(self, address, handler)
def __init__(self, origin, branch="master", path=".", mount_point="."): super(GitFS, self).__init__() self.origin = origin self.branch = branch self.root = os.path.realpath(path) self.mount_point = mount_point self.halt = False self.rwlock = Lock() self.need_sync_time = None # Can't use the default rlock here since we want to acquire/release from different threads self.sync_c = Condition(Lock()) self.timer_c = Condition(Lock()) self.id = None self.timer = None self.handlers = { "ping": self._handlePing, "lock": self._handleLock, "unlock": self._handleUnlock, "info": self._handleInfo, "getConfig": self._getConfig, } self.lock_timer = None self.lock_lock = Condition() self.locks = {} self.lock_expire_time = time() self.control_dir = self.getControlDirectory() try: os.makedirs(self.control_dir) except OSError: pass self.info_dir = self.getInfoDirectory(self.root) try: os.makedirs(self.info_dir) except OSError: pass self.control_socket_path = self.getControlSocketPath(self.getID(), server=True) self.lockGitFSDir() try: try: client = GitFSClient.getClientByPath(self.mount_point, False, False) raise FuseOSError(EBUSY) except GitFSError: pass try: os.remove(self.control_socket_path) except OSError: pass self.control_server = None self.control_server = ThreadingUnixStreamServer( self.control_socket_path, type( "GitFSRequestHandler", (PacketizeMixIn, BaseRequestHandler, object), dict( fs=self, dictFromString=self.parseDict, stringFromDict=self.marshalDict, handleDict=lambda s, d: s.fs._handleRequest(s, d), ), ), ) self.control_server.daemon_threads = True # setup the threads last so that they don't prevent an exit. self.control_thread = Thread(target=self.control_server.serve_forever, args=()) self.control_thread.start() finally: self.unlockGitFSDir() mt = self.getMTab() mt[mount_point] = self.getID() self.updateMTab(mt) self.repo = GitRepo(path, origin, branch, sync=True) self.sync_thread = Thread(target=self._sync, args=()) self.sync_thread.start()
class GitFS(GitFSBase, Operations): """A simple filesystem using Git and FUSE. """ def __init__(self, origin, branch="master", path=".", mount_point="."): super(GitFS, self).__init__() self.origin = origin self.branch = branch self.root = os.path.realpath(path) self.mount_point = mount_point self.halt = False self.rwlock = Lock() self.need_sync_time = None # Can't use the default rlock here since we want to acquire/release from different threads self.sync_c = Condition(Lock()) self.timer_c = Condition(Lock()) self.id = None self.timer = None self.handlers = { "ping": self._handlePing, "lock": self._handleLock, "unlock": self._handleUnlock, "info": self._handleInfo, "getConfig": self._getConfig, } self.lock_timer = None self.lock_lock = Condition() self.locks = {} self.lock_expire_time = time() self.control_dir = self.getControlDirectory() try: os.makedirs(self.control_dir) except OSError: pass self.info_dir = self.getInfoDirectory(self.root) try: os.makedirs(self.info_dir) except OSError: pass self.control_socket_path = self.getControlSocketPath(self.getID(), server=True) self.lockGitFSDir() try: try: client = GitFSClient.getClientByPath(self.mount_point, False, False) raise FuseOSError(EBUSY) except GitFSError: pass try: os.remove(self.control_socket_path) except OSError: pass self.control_server = None self.control_server = ThreadingUnixStreamServer( self.control_socket_path, type( "GitFSRequestHandler", (PacketizeMixIn, BaseRequestHandler, object), dict( fs=self, dictFromString=self.parseDict, stringFromDict=self.marshalDict, handleDict=lambda s, d: s.fs._handleRequest(s, d), ), ), ) self.control_server.daemon_threads = True # setup the threads last so that they don't prevent an exit. self.control_thread = Thread(target=self.control_server.serve_forever, args=()) self.control_thread.start() finally: self.unlockGitFSDir() mt = self.getMTab() mt[mount_point] = self.getID() self.updateMTab(mt) self.repo = GitRepo(path, origin, branch, sync=True) self.sync_thread = Thread(target=self._sync, args=()) self.sync_thread.start() def getID(self): if self.id is None: self.id = mUUID.getUUIDFromFile(self.getUUIDFile(self.root), create=True).toString() return self.id def _lockWithTimeOut(self, name, t): if t <= 0: return self.lock_lock.acquire() expt = t + time() self.locks[name] = expt if self.lock_expire_time - expt < 0: self.lock_expire_time = expt if self.lock_timer != None: self.lock_timer.cancel() else: logging.debug("Aquiring fresh lock") self.sync_c.acquire() self.lock_timer = Timer(t, self._lockTimerExpire, args=()) self.lock_timer.start() self.lock_lock.release() def _lockTimerExpire(self): logging.debug("_lockTimeExpire") self.lock_lock.acquire() self.__lockTimerExpire() self.lock_lock.release() def __lockTimerExpire(self): logging.debug("__lockTimeExpire") now = time() t = now for key in self.locks: if t - self.locks[key] < 0: t = self.locks[key] if now - self.locks[key] > 300: del self.locks[key] if t - now > 0: self.lock_expire_time = t if self.lock_timer != None: self.lock_timer.cancel() else: logging.debug("***** ERROR ***** __lockTimerExpire doesn't have lock. acquiring") self.sync_c.acquire() self.lock_timer = Timer(t - now, self._lockTimerExpire, args=()) self.lock_timer.start() logging.debug("extending lock.") else: if self.lock_timer != None: logging.debug("releasing lock.") self.lock_timer.cancel() self.lock_timer = None self.sync_c.release() def _unlock(self, name): if name not in self.locks: return self.lock_lock.acquire() t = self.locks[name] del self.locks[name] if t >= self.lock_expire_time or len(keys(self.locks)) == 0: self.__lockTimerExpire() self.lock_lock.release() def _handleRequest(self, request, d): if d["action"] in self.handlers: mf = self.handlers[d["action"]] return mf(d, request) logging.debug("No request in packet: %s" % d) self._respond(request, {"status": "Unknown Command"}) return None def _respond(self, request, responseDict): request.sendDict(responseDict) def _handlePing(self, reqDict, request): self._respond(request, {"status": "ok", "message": "pong"}) def _handleLock(self, reqDict, request): self._lockWithTimeOut("%s" % request.request.fileno(), 60) self._respond(request, {"status": "ok", "name": "%s" % request.request.fileno()}) def _handleUnlock(self, reqDict, request): self._unlock("%s" % request.request.fileno()) self._respond(request, {"status": "ok"}) def _handleInfo(self, reqDict, request): self._respond( request, { "status": "ok", "origin": self.repo.origin, "branch": self.repo.branch, "root": self.root, "path": self.mount_point, }, ) def _getConfig(self, reqDict, request): key = reqDict["key"] resp = self.getConfigForInstance(key) self._respond(request, {"status": "ok", key: resp}) def _sync(self): while True: self.sync_c.acquire() if not self.halt: # wait till a sync request comes self.sync_c.wait() self.timer_c.acquire() if self.timer != None: self.timer.cancel() self.timer = None self.timer_c.release() try: if not self.repo.synchronize(): # sync failed, we need to try again. self.needSync() self.need_sync_time = None except Exception as e: logging.debug("synchronize threw exception %s" % e) self.sync_c.release() # can't release this until sync is complete because we can't change files while we sync. else: self.repo.forcePush() self.repo.push() self.sync_c.release() break def getHostInfo(): if self.hostinfo is None: self.hostinfo = HostInfo() self.hostinfo.update() return self.hostinfo def matchesInstance(self, key): """ returns a number determinging how well the past in key matches the current instance.""" if key == "default": return 0.1 # Very weak match hostinfo = self.getHostInfo() try: ip = socket.inet_pton(key) return hostinfo.matchAddress(ip) except SocketError: pass return hostinfo.matchHostName(key) def getConfigForInstanceSingleFile(self, key, name): logging.debug("getConfigForInstanceSingleFile(%s, %s)" % (key, name)) c = self.getConfig(name) if c is None: return None match = 0 if key not in c: return None v = c[key] logging.debug("getConfigForInstanceSingleFile(%s, %s) value=%s" % (key, name, v)) if isinstance(v, dict): rv = None for (key, value) in v.iteritems(): tm = self.matchesInstance(key) if tm > match: match = tm rv = value v = rv if v is None: return None if isinstance(v, list): v = random.choose(v) return (match, v) def getConfigForInstance(self, key): match = 0 value = None logging.debug("getConfigForInstance(%s)" % key) if key in self.config_file_priorities: filenames = self.config_file_priorities[key] else: filenames = self.config_file_priorities["default"] for name in filenames: a = self.getConfigForInstanceSingleFile(key, name) if a is None: continue (m, v) = a if m >= match: match = m value = v return value def forceSync(self): logging.debug("forceSync()") self.sync_c.acquire() self.sync_c.notifyAll() self.sync_c.release() def needSync(self): logging.debug("needSync()") self.timer_c.acquire() if self.need_sync_time is None: self.need_sync_time = time() if self.timer != None: if time() - self.need_sync_time > 5 * 60: return self.timer.cancel() # don't do anything until there is a pause. self.timer = Timer(10, self.forceSync, args=()) self.timer.start() self.timer_c.release() def shutdown(self): # stop sync thread self.sync_c.acquire() self.halt = True self.sync_c.notifyAll() self.sync_c.release() self.repo.shutDown() def destroy(self, path): self.shutdown() if self.control_server != None: self.control_server.shutdown() try: os.remove(self.control_socket_path) except OSError: pass def __call__(self, op, path, *args): try: logging.debug("calling %s on %s" % (op, path)) path = self.escapePath(path) r = super(GitFS, self).__call__(op, self.root + path, *args) pr = "%s" % r pr = pr[:10] logging.debug("returning %s for %s" % (pr, op)) return r except Exception as e: logging.debug("Unhandled exception %s" % e) raise e def access(self, path, mode): if not os.access(path, mode): raise FuseOSError(EACCES) def chmod(self, path, mode): self.needSync() return os.chmod(path, mode) def chown(self, path, uid, gid): self.needSync() return super(GitFS, self).chown(path, uid, gid) def create(self, path, mode): # XXXXX Fixme what should the flags be? return os.open(path, os.O_RDWR | os.O_CREAT, mode) def flush(self, path, fh): self.sync_c.acquire() self.sync_c.notifyAll() self.sync_c.release() return os.fsync(fh) def fsync(self, path, datasync, fh): self.needSync() return os.fsync(fh) def fsyncdir(self, path, datasync, fh): return self.fsync(path, datasync, fh) def getattr(self, path, fh=None): st = os.lstat(path) return dict( (key, getattr(st, key)) for key in ("st_atime", "st_ctime", "st_gid", "st_mode", "st_mtime", "st_nlink", "st_size", "st_uid") ) getxattr = None def link(self, target, source): source = self.escapePath(source) self.needSync() return os.link(source, target) listxattr = None mknod = os.mknod def mkdir(self, path, mode): return os.mkdir(path, mode) def open(self, path, fip): f = os.open(path, fip) logging.debug("open(%s, %s): %d" % (path, fip, f)) return f def read(self, path, size, offset, fh): with self.rwlock: os.lseek(fh, offset, 0) return os.read(fh, size) def readdir(self, path, fh): files = os.listdir(path) uefiles = [] for file in files: if self.isValidPath(file): uefiles = uefiles + [self.unescapePath(file)] return [".", ".."] + uefiles readlink = os.readlink def release(self, path, fh): return os.close(fh) def rename(self, old, new): self.needSync() return os.rename(old, self.root + self.escapePath(new)) def rmdir(self, path): self.needSync() return os.rmdir(path) def statfs(self, path): stv = os.statvfs(path) return dict( (key, getattr(stv, key)) for key in ( "f_bavail", "f_bfree", "f_blocks", "f_bsize", "f_favail", "f_ffree", "f_files", "f_flag", "f_frsize", "f_namemax", ) ) def symlink(self, target, source): self.needSync() return os.symlink(source, target) def truncate(self, path, length, fh=None): self.needSync() with open(path, "r+") as f: f.truncate(length) def unlink(self, path): self.needSync() return os.unlink(path) utimens = os.utime def write(self, path, data, offset, fh): self.needSync() with self.rwlock: os.lseek(fh, offset, 0) return os.write(fh, data)