def __init__(self, origin, branch="master", path="."):
self.origin = origin
self.branch = branch
self.root = os.path.realpath(path)
self.halt = False
self.rwlock = Lock()
# Can't use the default rlock here since we want to acquire/release from different threads
self.sync_c = Condition(Lock())
self.timer_c = Condition(Lock())
self.timer = None
self.handlers = {
"ping": self._handlePing,
"lock": self._handleLock,
"unlock": self._handleUnlock,
"info": self._handleInfo,
}
self.lock_timer = None
self.lock_lock = Condition()
self.locks = {}
self.lock_expire_time = time()
self.control_dir = self.getControlDirectory(self.root)
try:
os.mkdir(self.control_dir)
except OSError:
pass
self.control_socket_path = self.getControlSocketPath(self.root)
client = GitFSClient(self.root)
self.needSyncTime = None
try:
if client.pingRemote():
# There is another file system mounted.
logging.debug("Exiting because file system already mounted.\n")
raise FuseOSError(EBUSY)
except socket.error as se:
logging.debug("socket.error = %s" % se)
pass
client.close()
client = None
try:
os.remove(self.control_socket_path)
except OSError:
pass
self.control_server = None
self.control_server = ThreadingUnixStreamServer(
self.control_socket_path,
type(
"GitFSRequestHandler",
(PacketizeMixIn, BaseRequestHandler, object),
dict(
fs=self,
dictFromString=self.parseDict,
stringFromDict=self.marshalDict,
handleDict=lambda s, d: s.fs._handleRequest(s, d),
),
),
)
self.control_server.daemon_threads = True
# setup the threads last so that they don't prevent an exit.
self.control_thread = Thread(target=self.control_server.serve_forever, args=())
self.control_thread.start()
self.repo = GitRepo(path, origin, branch, sync=True)
self.sync_thread = Thread(target=self._sync, args=())
self.sync_thread.start()
def start_server():
server = ThreadingUnixStreamServer(unix_socket_path(),
RequestHandler)
server.serve_forever()
class GitFS(GitFSStringMixIn, Operations):
"""A simple filesystem using Git and FUSE.
"""
def __init__(self, origin, branch="master", path="."):
self.origin = origin
self.branch = branch
self.root = os.path.realpath(path)
self.halt = False
self.rwlock = Lock()
# Can't use the default rlock here since we want to acquire/release from different threads
self.sync_c = Condition(Lock())
self.timer_c = Condition(Lock())
self.timer = None
self.handlers = {
"ping": self._handlePing,
"lock": self._handleLock,
"unlock": self._handleUnlock,
"info": self._handleInfo,
}
self.lock_timer = None
self.lock_lock = Condition()
self.locks = {}
self.lock_expire_time = time()
self.control_dir = self.getControlDirectory(self.root)
try:
os.mkdir(self.control_dir)
except OSError:
pass
self.control_socket_path = self.getControlSocketPath(self.root)
client = GitFSClient(self.root)
self.needSyncTime = None
try:
if client.pingRemote():
# There is another file system mounted.
logging.debug("Exiting because file system already mounted.\n")
raise FuseOSError(EBUSY)
except socket.error as se:
logging.debug("socket.error = %s" % se)
pass
client.close()
client = None
try:
os.remove(self.control_socket_path)
except OSError:
pass
self.control_server = None
self.control_server = ThreadingUnixStreamServer(
self.control_socket_path,
type(
"GitFSRequestHandler",
(PacketizeMixIn, BaseRequestHandler, object),
dict(
fs=self,
dictFromString=self.parseDict,
stringFromDict=self.marshalDict,
handleDict=lambda s, d: s.fs._handleRequest(s, d),
),
),
)
self.control_server.daemon_threads = True
# setup the threads last so that they don't prevent an exit.
self.control_thread = Thread(target=self.control_server.serve_forever, args=())
self.control_thread.start()
self.repo = GitRepo(path, origin, branch, sync=True)
self.sync_thread = Thread(target=self._sync, args=())
self.sync_thread.start()
def _lockWithTimeOut(self, name, t):
if t <= 0:
return
self.lock_lock.acquire()
expt = t + time()
self.locks[name] = expt
if self.lock_expire_time - expt < 0:
self.lock_expire_time = expt
if self.lock_timer != None:
self.lock_timer.cancel()
else:
logging.debug("Aquiring fresh lock")
self.sync_c.acquire()
self.lock_timer = Timer(t, self._lockTimerExpire, args=())
self.lock_timer.start()
self.lock_lock.release()
def _lockTimerExpire(self):
logging.debug("_lockTimeExpire")
self.lock_lock.acquire()
self.__lockTimerExpire()
self.lock_lock.release()
def __lockTimerExpire(self):
logging.debug("__lockTimeExpire")
now = time()
t = now
for key in self.locks:
if t - self.locks[key] < 0:
t = self.locks[key]
if now - self.locks[key] > 300:
del self.locks[key]
if t - now > 0:
self.lock_expire_time = t
if self.lock_timer != None:
self.lock_timer.cancel()
else:
logging.debug("***** ERROR ***** __lockTimerExpire doesn't have lock. acquiring")
self.sync_c.acquire()
self.lock_timer = Timer(t - now, self._lockTimerExpire, args=())
self.lock_timer.start()
logging.debug("extending lock.")
else:
if self.lock_timer != None:
logging.debug("releasing lock.")
self.lock_timer.cancel()
self.lock_timer = None
self.sync_c.release()
def _unlock(self, name):
if name not in self.locks:
return
self.lock_lock.acquire()
t = self.locks[name]
del self.locks[name]
if t >= self.lock_expire_time or len(keys(self.locks)) == 0:
self.__lockTimerExpire()
self.lock_lock.release()
def _handleRequest(self, request, d):
if d["action"] in self.handlers:
mf = self.handlers[d["action"]]
return mf(d, request)
logging.debug("No request in packet: %s" % d)
self._respond(request, {"status": "Unknown Command"})
return None
def _respond(self, request, responseDict):
request.sendDict(responseDict)
def _handlePing(self, reqDict, request):
self._respond(request, {"status": "ok", "message": "pong"})
def _handleLock(self, reqDict, request):
self._lockWithTimeOut("%s" % request.request.fileno(), 60)
self._respond(request, {"status": "ok", "name": "%s" % request.request.fileno()})
def _handleUnlock(self, reqDict, request):
self._unlock("%s" % request.request.fileno())
self._respond(request, {"status": "ok"})
def _handleInfo(self, reqDict, request):
self._respond(request, {"status": "ok", "origin": self.repo.origin, "branch": self.repo.branch})
def _sync(self):
while True:
self.sync_c.acquire()
if not self.halt:
# wait till a sync request comes
self.sync_c.wait()
self.timer_c.acquire()
if self.timer != None:
self.timer.cancel()
self.timer = None
self.timer_c.release()
try:
if not self.repo.synchronize():
# sync failed, we need to try again.
self.needSync()
self.needSyncTime = None
except Exception as e:
logging.debug("synchronize threw exception %s" % e)
self.sync_c.release() # can't release this until sync is complete because we can't change files while we sync.
else:
self.repo.forcePush()
self.repo.push()
self.sync_c.release()
break
def forceSync(self):
logging.debug("forceSync()")
self.sync_c.acquire()
self.sync_c.notifyAll()
self.sync_c.release()
def needSync(self):
logging.debug("needSync()")
self.timer_c.acquire()
if self.needSyncTime == None:
self.needSyncTime = time()
if self.timer != None:
if time() - self.needSyncTime > 5 * 60:
return
self.timer.cancel()
# don't do anything until there is a pause.
self.timer = Timer(10, self.forceSync, args=())
self.timer.start()
self.timer_c.release()
def shutdown(self):
# stop sync thread
self.sync_c.acquire()
self.halt = True
self.sync_c.notifyAll()
self.sync_c.release()
self.repo.shutDown()
def destroy(self, path):
self.shutdown()
if self.control_server != None:
self.control_server.shutdown()
try:
os.remove(self.control_socket_path)
except OSError:
pass
def __call__(self, op, path, *args):
try:
logging.debug("calling %s on %s" % (op, path))
path = self.escapePath(path)
r = super(GitFS, self).__call__(op, self.root + path, *args)
pr = "%s" % r
pr = pr[:10]
logging.debug("returning %s for %s" % (pr, op))
return r
except Exception as e:
logging.debug("Unhandled exception %s" % e)
raise e
def access(self, path, mode):
if not os.access(path, mode):
raise FuseOSError(EACCES)
def chmod(self, path, mode):
self.needSync()
return os.chmod(path, mode)
def chown(self, path, uid, gid):
self.needSync()
return super(GitFS, self).chown(path, uid, gid)
def create(self, path, mode):
# XXXXX Fixme what should the flags be?
return os.open(path, os.O_RDWR | os.O_CREAT, mode)
def flush(self, path, fh):
self.sync_c.acquire()
self.sync_c.notifyAll()
self.sync_c.release()
return os.fsync(fh)
def fsync(self, path, datasync, fh):
self.needSync()
return os.fsync(fh)
def fsyncdir(self, path, datasync, fh):
return self.fsync(path, datasync, fh)
def getattr(self, path, fh=None):
st = os.lstat(path)
return dict(
(key, getattr(st, key))
for key in ("st_atime", "st_ctime", "st_gid", "st_mode", "st_mtime", "st_nlink", "st_size", "st_uid")
)
getxattr = None
def link(self, target, source):
source = self.escapePath(source)
self.needSync()
return os.link(source, target)
listxattr = None
mknod = os.mknod
def mkdir(self, path, mode):
return os.mkdir(path, mode)
def open(self, path, fip):
f = os.open(path, fip)
logging.debug("open(%s, %s): %d" % (path, fip, f))
return f
def read(self, path, size, offset, fh):
with self.rwlock:
os.lseek(fh, offset, 0)
return os.read(fh, size)
def readdir(self, path, fh):
files = os.listdir(path)
uefiles = []
for file in files:
if self.isValidPath(file):
uefiles = uefiles + [self.unescapePath(file)]
return [".", ".."] + uefiles
readlink = os.readlink
def release(self, path, fh):
return os.close(fh)
def rename(self, old, new):
self.needSync()
return os.rename(old, self.root + self.escapePath(new))
def rmdir(self, path):
self.needSync()
return os.rmdir(path)
def statfs(self, path):
stv = os.statvfs(path)
return dict(
(key, getattr(stv, key))
for key in (
"f_bavail",
"f_bfree",
"f_blocks",
"f_bsize",
"f_favail",
"f_ffree",
"f_files",
"f_flag",
"f_frsize",
"f_namemax",
)
)
def symlink(self, target, source):
self.needSync()
return os.symlink(source, target)
def truncate(self, path, length, fh=None):
self.needSync()
with open(path, "r+") as f:
f.truncate(length)
def unlink(self, path):
self.needSync()
return os.unlink(path)
utimens = os.utime
def write(self, path, data, offset, fh):
self.needSync()
with self.rwlock:
os.lseek(fh, offset, 0)
return os.write(fh, data)
def __init__(self, origin, branch='master', path='.', mount_point='.'):
super(GitFS, self).__init__()
self.origin = origin
self.branch = branch
self.root = os.path.realpath(path)
self.mount_point = mount_point
self.halt = False
self.rwlock = Lock()
self.need_sync_time = None
# Can't use the default rlock here since we want to acquire/release from different threads
self.sync_c = Condition(Lock())
self.timer_c = Condition(Lock())
self.id = None
self.timer = None
self.handlers = {
'ping': self._handlePing,
'lock': self._handleLock,
'unlock': self._handleUnlock,
'info': self._handleInfo,
'getConfig': self._getConfig
}
self.lock_timer = None
self.lock_lock = Condition()
self.locks = {}
self.lock_expire_time = time()
self.control_dir = self.getControlDirectory()
try:
os.makedirs(self.control_dir)
except OSError:
pass
self.info_dir = self.getInfoDirectory(self.root)
try:
os.makedirs(self.info_dir)
except OSError:
pass
self.control_socket_path = self.getControlSocketPath(self.getID(),
server=True)
self.lockGitFSDir()
try:
try:
client = GitFSClient.getClientByPath(self.mount_point, False,
False)
raise FuseOSError(EBUSY)
except GitFSError:
pass
try:
os.remove(self.control_socket_path)
except OSError:
pass
self.control_server = None
self.control_server = ThreadingUnixStreamServer(
self.control_socket_path,
type(
"GitFSRequestHandler",
(PacketizeMixIn, BaseRequestHandler, object),
dict(fs=self,
dictFromString=self.parseDict,
stringFromDict=self.marshalDict,
handleDict=lambda s, d: s.fs._handleRequest(s, d))))
self.control_server.daemon_threads = True
# setup the threads last so that they don't prevent an exit.
self.control_thread = Thread(
target=self.control_server.serve_forever, args=())
self.control_thread.start()
finally:
self.unlockGitFSDir()
mt = self.getMTab()
mt[mount_point] = self.getID()
self.updateMTab(mt)
self.repo = GitRepo(path, origin, branch, sync=True)
self.sync_thread = Thread(target=self._sync, args=())
self.sync_thread.start()
# -*- coding: utf-8 -*-
""" 使用python标准库的 ThreadingTCPServer
: 如果想用gevent, 可以用gevent patch all
"""
from SocketServer import ThreadingUnixStreamServer, StreamRequestHandler
from mprpc import RPCHandler
class SumHandler(RPCHandler):
def sum(self, x, y):
return x + y
class _Handler(StreamRequestHandler):
def handle(self):
SumHandler()(self.connection, None)
server = ThreadingUnixStreamServer('./x.sock', _Handler)
server.serve_forever()
sys.exit()
sys.stdout.flush()
sys.stderr.flush()
si = file('/dev/null', 'r')
so = file('/dev/null', 'a+')
se = file('/dev/null', 'a+', 0)
os.dup2(si.fileno(), sys.stdin.fileno())
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())
if __name__ == '__main__':
logger = configure_logging()
# Socket server creates its own socket file. Delete if it exists already.
if os.path.exists(SOCKET_FILENAME):
logger.warning('Unlinking existing socket: %s' % SOCKET_FILENAME)
os.unlink(SOCKET_FILENAME)
server = ThreadingUnixStreamServer(SOCKET_FILENAME, GitHubPingHandler)
os.chmod(SOCKET_FILENAME, 0777)
try:
logger.info('Starting Gitte socket server at: %s' % SOCKET_FILENAME)
make_service()
server.serve_forever()
except KeyboardInterrupt:
os.unlink(SOCKET_FILENAME)
print "\nKeyboard interupt recieved, Gitte server stopping..."
class GitFS(GitFSBase, Operations):
"""A simple filesystem using Git and FUSE.
"""
def __init__(self, origin, branch='master', path='.', mount_point='.'):
super(GitFS, self).__init__()
self.origin = origin
self.branch = branch
self.root = os.path.realpath(path)
self.mount_point = mount_point
self.halt = False
self.rwlock = Lock()
self.need_sync_time = None
# Can't use the default rlock here since we want to acquire/release from different threads
self.sync_c = Condition(Lock())
self.timer_c = Condition(Lock())
self.id = None
self.timer = None
self.handlers = {
'ping': self._handlePing,
'lock': self._handleLock,
'unlock': self._handleUnlock,
'info': self._handleInfo,
'getConfig': self._getConfig
}
self.lock_timer = None
self.lock_lock = Condition()
self.locks = {}
self.lock_expire_time = time()
self.control_dir = self.getControlDirectory()
try:
os.makedirs(self.control_dir)
except OSError:
pass
self.info_dir = self.getInfoDirectory(self.root)
try:
os.makedirs(self.info_dir)
except OSError:
pass
self.control_socket_path = self.getControlSocketPath(self.getID(),
server=True)
self.lockGitFSDir()
try:
try:
client = GitFSClient.getClientByPath(self.mount_point, False,
False)
raise FuseOSError(EBUSY)
except GitFSError:
pass
try:
os.remove(self.control_socket_path)
except OSError:
pass
self.control_server = None
self.control_server = ThreadingUnixStreamServer(
self.control_socket_path,
type(
"GitFSRequestHandler",
(PacketizeMixIn, BaseRequestHandler, object),
dict(fs=self,
dictFromString=self.parseDict,
stringFromDict=self.marshalDict,
handleDict=lambda s, d: s.fs._handleRequest(s, d))))
self.control_server.daemon_threads = True
# setup the threads last so that they don't prevent an exit.
self.control_thread = Thread(
target=self.control_server.serve_forever, args=())
self.control_thread.start()
finally:
self.unlockGitFSDir()
mt = self.getMTab()
mt[mount_point] = self.getID()
self.updateMTab(mt)
self.repo = GitRepo(path, origin, branch, sync=True)
self.sync_thread = Thread(target=self._sync, args=())
self.sync_thread.start()
def getID(self):
if self.id is None:
self.id = mUUID.getUUIDFromFile(self.getUUIDFile(self.root),
create=True).toString()
return self.id
def _lockWithTimeOut(self, name, t):
if t <= 0:
return
self.lock_lock.acquire()
expt = t + time()
self.locks[name] = expt
if self.lock_expire_time - expt < 0:
self.lock_expire_time = expt
if self.lock_timer != None:
self.lock_timer.cancel()
else:
logging.debug("Aquiring fresh lock")
self.sync_c.acquire()
self.lock_timer = Timer(t, self._lockTimerExpire, args=())
self.lock_timer.start()
self.lock_lock.release()
def _lockTimerExpire(self):
logging.debug('_lockTimeExpire')
self.lock_lock.acquire()
self.__lockTimerExpire()
self.lock_lock.release()
def __lockTimerExpire(self):
logging.debug('__lockTimeExpire')
now = time()
t = now
for key in self.locks:
if t - self.locks[key] < 0:
t = self.locks[key]
if now - self.locks[key] > 300:
del self.locks[key]
if t - now > 0:
self.lock_expire_time = t
if self.lock_timer != None:
self.lock_timer.cancel()
else:
logging.debug(
"***** ERROR ***** __lockTimerExpire doesn't have lock. acquiring"
)
self.sync_c.acquire()
self.lock_timer = Timer(t - now, self._lockTimerExpire, args=())
self.lock_timer.start()
logging.debug("extending lock.")
else:
if self.lock_timer != None:
logging.debug("releasing lock.")
self.lock_timer.cancel()
self.lock_timer = None
self.sync_c.release()
def _unlock(self, name):
if name not in self.locks:
return
self.lock_lock.acquire()
t = self.locks[name]
del self.locks[name]
if t >= self.lock_expire_time or len(keys(self.locks)) == 0:
self.__lockTimerExpire()
self.lock_lock.release()
def _handleRequest(self, request, d):
if d['action'] in self.handlers:
mf = self.handlers[d['action']]
return mf(d, request)
logging.debug("No request in packet: %s" % d)
self._respond(request, {'status': 'Unknown Command'})
return None
def _respond(self, request, responseDict):
request.sendDict(responseDict)
def _handlePing(self, reqDict, request):
self._respond(request, {'status': 'ok', 'message': 'pong'})
def _handleLock(self, reqDict, request):
self._lockWithTimeOut('%s' % request.request.fileno(), 60)
self._respond(request, {
'status': 'ok',
'name': '%s' % request.request.fileno()
})
def _handleUnlock(self, reqDict, request):
self._unlock('%s' % request.request.fileno())
self._respond(request, {'status': 'ok'})
def _handleInfo(self, reqDict, request):
self._respond(
request, {
'status': 'ok',
'origin': self.repo.origin,
'branch': self.repo.branch,
'root': self.root,
'path': self.mount_point
})
def _getConfig(self, reqDict, request):
key = reqDict['key']
resp = self.getConfigForInstance(key)
self._respond(request, {'status': 'ok', key: resp})
def _sync(self):
while True:
self.sync_c.acquire()
if not self.halt:
# wait till a sync request comes
self.sync_c.wait()
self.timer_c.acquire()
if self.timer != None:
self.timer.cancel()
self.timer = None
self.timer_c.release()
try:
if not self.repo.synchronize():
#sync failed, we need to try again.
self.needSync()
self.need_sync_time = None
except Exception as e:
logging.debug("synchronize threw exception %s" % e)
self.sync_c.release(
) # can't release this until sync is complete because we can't change files while we sync.
else:
self.repo.forcePush()
self.repo.push()
self.sync_c.release()
break
def getHostInfo():
if self.hostinfo is None:
self.hostinfo = HostInfo()
self.hostinfo.update()
return self.hostinfo
def matchesInstance(self, key):
""" returns a number determinging how well the past in key matches the current instance."""
if key == 'default':
return .1 #Very weak match
hostinfo = self.getHostInfo()
try:
ip = socket.inet_pton(key)
return hostinfo.matchAddress(ip)
except SocketError:
pass
return hostinfo.matchHostName(key)
def getConfigForInstanceSingleFile(self, key, name):
logging.debug('getConfigForInstanceSingleFile(%s, %s)' % (key, name))
c = self.getConfig(name)
if c is None:
return None
match = 0
if key not in c:
return None
v = c[key]
logging.debug('getConfigForInstanceSingleFile(%s, %s) value=%s' %
(key, name, v))
if isinstance(v, dict):
rv = None
for (key, value) in v.iteritems():
tm = self.matchesInstance(key)
if tm > match:
match = tm
rv = value
v = rv
if v is None:
return None
if isinstance(v, list):
v = random.choose(v)
return (match, v)
def getConfigForInstance(self, key):
match = 0
value = None
logging.debug('getConfigForInstance(%s)' % key)
if key in self.config_file_priorities:
filenames = self.config_file_priorities[key]
else:
filenames = self.config_file_priorities['default']
for name in filenames:
a = self.getConfigForInstanceSingleFile(key, name)
if a is None:
continue
(m, v) = a
if (m >= match):
match = m
value = v
return value
def forceSync(self):
logging.debug('forceSync()')
self.sync_c.acquire()
self.sync_c.notifyAll()
self.sync_c.release()
def needSync(self):
logging.debug('needSync()')
self.timer_c.acquire()
if self.need_sync_time is None:
self.need_sync_time = time()
if self.timer != None:
if time() - self.need_sync_time > 5 * 60:
return
self.timer.cancel()
# don't do anything until there is a pause.
self.timer = Timer(10, self.forceSync, args=())
self.timer.start()
self.timer_c.release()
def shutdown(self):
# stop sync thread
self.sync_c.acquire()
self.halt = True
self.sync_c.notifyAll()
self.sync_c.release()
self.repo.shutDown()
def destroy(self, path):
self.shutdown()
if self.control_server != None:
self.control_server.shutdown()
try:
os.remove(self.control_socket_path)
except OSError:
pass
def __call__(self, op, path, *args):
try:
logging.debug("calling %s on %s" % (op, path))
path = self.escapePath(path)
r = super(GitFS, self).__call__(op, self.root + path, *args)
pr = "%s" % r
pr = pr[:10]
logging.debug("returning %s for %s" % (pr, op))
return r
except Exception as e:
logging.debug("Unhandled exception %s" % e)
raise e
def access(self, path, mode):
if not os.access(path, mode):
raise FuseOSError(EACCES)
def chmod(self, path, mode):
self.needSync()
return os.chmod(path, mode)
def chown(self, path, uid, gid):
self.needSync()
return super(GitFS, self).chown(path, uid, gid)
def create(self, path, mode):
# XXXXX Fixme what should the flags be?
return os.open(path, os.O_RDWR | os.O_CREAT, mode)
def flush(self, path, fh):
self.sync_c.acquire()
self.sync_c.notifyAll()
self.sync_c.release()
return os.fsync(fh)
def fsync(self, path, datasync, fh):
self.needSync()
return os.fsync(fh)
def fsyncdir(self, path, datasync, fh):
return self.fsync(path, datasync, fh)
def getattr(self, path, fh=None):
st = os.lstat(path)
return dict((key, getattr(st, key))
for key in ('st_atime', 'st_ctime', 'st_gid', 'st_mode',
'st_mtime', 'st_nlink', 'st_size', 'st_uid'))
getxattr = None
def link(self, target, source):
source = self.escapePath(source)
self.needSync()
return os.link(source, target)
listxattr = None
mknod = os.mknod
def mkdir(self, path, mode):
return os.mkdir(path, mode)
def open(self, path, fip):
f = os.open(path, fip)
logging.debug("open(%s, %s): %d" % (path, fip, f))
return f
def read(self, path, size, offset, fh):
with self.rwlock:
os.lseek(fh, offset, 0)
return os.read(fh, size)
def readdir(self, path, fh):
files = os.listdir(path)
uefiles = []
for file in files:
if self.isValidPath(file):
uefiles = uefiles + [self.unescapePath(file)]
return ['.', '..'] + uefiles
readlink = os.readlink
def release(self, path, fh):
return os.close(fh)
def rename(self, old, new):
self.needSync()
return os.rename(old, self.root + self.escapePath(new))
def rmdir(self, path):
self.needSync()
return os.rmdir(path)
def statfs(self, path):
stv = os.statvfs(path)
return dict((key, getattr(stv, key))
for key in ('f_bavail', 'f_bfree', 'f_blocks', 'f_bsize',
'f_favail', 'f_ffree', 'f_files', 'f_flag',
'f_frsize', 'f_namemax'))
def symlink(self, target, source):
self.needSync()
return os.symlink(source, target)
def truncate(self, path, length, fh=None):
self.needSync()
with open(path, 'r+') as f:
f.truncate(length)
def unlink(self, path):
self.needSync()
return os.unlink(path)
utimens = os.utime
def write(self, path, data, offset, fh):
self.needSync()
with self.rwlock:
os.lseek(fh, offset, 0)
return os.write(fh, data)
def return_unix(sock='spamd.sock'):
"""Return a unix SPAMD server"""
if os.path.exists(sock):
os.remove(sock)
server = ThreadingUnixStreamServer(sock, TestSpamdHandler)
return server
def __init__(self, address, handler, cqueue):
self.queue = cqueue
ThreadingUnixStreamServer.__init__(self, address, handler)
sys.exit()
sys.stdout.flush()
sys.stderr.flush()
si = file('/dev/null', 'r')
so = file('/dev/null', 'a+')
se = file('/dev/null', 'a+', 0)
os.dup2(si.fileno(), sys.stdin.fileno())
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())
if __name__ == '__main__':
logger = configure_logging()
# Socket server creates its own socket file. Delete if it exists already.
if os.path.exists(SOCKET_FILENAME):
logger.warning('Unlinking existing socket: %s' % SOCKET_FILENAME)
os.unlink(SOCKET_FILENAME)
server = ThreadingUnixStreamServer(SOCKET_FILENAME, GitHubPingHandler)
os.chmod(SOCKET_FILENAME, 0777)
try:
logger.info('Starting Gitte socket server at: %s' % SOCKET_FILENAME)
make_service()
server.serve_forever()
except KeyboardInterrupt:
os.unlink(SOCKET_FILENAME)
print "\nKeyboard interupt recieved, Gitte server stopping..."
def __init__(self, origin, branch="master", path=".", mount_point="."):
super(GitFS, self).__init__()
self.origin = origin
self.branch = branch
self.root = os.path.realpath(path)
self.mount_point = mount_point
self.halt = False
self.rwlock = Lock()
self.need_sync_time = None
# Can't use the default rlock here since we want to acquire/release from different threads
self.sync_c = Condition(Lock())
self.timer_c = Condition(Lock())
self.id = None
self.timer = None
self.handlers = {
"ping": self._handlePing,
"lock": self._handleLock,
"unlock": self._handleUnlock,
"info": self._handleInfo,
"getConfig": self._getConfig,
}
self.lock_timer = None
self.lock_lock = Condition()
self.locks = {}
self.lock_expire_time = time()
self.control_dir = self.getControlDirectory()
try:
os.makedirs(self.control_dir)
except OSError:
pass
self.info_dir = self.getInfoDirectory(self.root)
try:
os.makedirs(self.info_dir)
except OSError:
pass
self.control_socket_path = self.getControlSocketPath(self.getID(), server=True)
self.lockGitFSDir()
try:
try:
client = GitFSClient.getClientByPath(self.mount_point, False, False)
raise FuseOSError(EBUSY)
except GitFSError:
pass
try:
os.remove(self.control_socket_path)
except OSError:
pass
self.control_server = None
self.control_server = ThreadingUnixStreamServer(
self.control_socket_path,
type(
"GitFSRequestHandler",
(PacketizeMixIn, BaseRequestHandler, object),
dict(
fs=self,
dictFromString=self.parseDict,
stringFromDict=self.marshalDict,
handleDict=lambda s, d: s.fs._handleRequest(s, d),
),
),
)
self.control_server.daemon_threads = True
# setup the threads last so that they don't prevent an exit.
self.control_thread = Thread(target=self.control_server.serve_forever, args=())
self.control_thread.start()
finally:
self.unlockGitFSDir()
mt = self.getMTab()
mt[mount_point] = self.getID()
self.updateMTab(mt)
self.repo = GitRepo(path, origin, branch, sync=True)
self.sync_thread = Thread(target=self._sync, args=())
self.sync_thread.start()
class GitFS(GitFSBase, Operations):
"""A simple filesystem using Git and FUSE.
"""
def __init__(self, origin, branch="master", path=".", mount_point="."):
super(GitFS, self).__init__()
self.origin = origin
self.branch = branch
self.root = os.path.realpath(path)
self.mount_point = mount_point
self.halt = False
self.rwlock = Lock()
self.need_sync_time = None
# Can't use the default rlock here since we want to acquire/release from different threads
self.sync_c = Condition(Lock())
self.timer_c = Condition(Lock())
self.id = None
self.timer = None
self.handlers = {
"ping": self._handlePing,
"lock": self._handleLock,
"unlock": self._handleUnlock,
"info": self._handleInfo,
"getConfig": self._getConfig,
}
self.lock_timer = None
self.lock_lock = Condition()
self.locks = {}
self.lock_expire_time = time()
self.control_dir = self.getControlDirectory()
try:
os.makedirs(self.control_dir)
except OSError:
pass
self.info_dir = self.getInfoDirectory(self.root)
try:
os.makedirs(self.info_dir)
except OSError:
pass
self.control_socket_path = self.getControlSocketPath(self.getID(), server=True)
self.lockGitFSDir()
try:
try:
client = GitFSClient.getClientByPath(self.mount_point, False, False)
raise FuseOSError(EBUSY)
except GitFSError:
pass
try:
os.remove(self.control_socket_path)
except OSError:
pass
self.control_server = None
self.control_server = ThreadingUnixStreamServer(
self.control_socket_path,
type(
"GitFSRequestHandler",
(PacketizeMixIn, BaseRequestHandler, object),
dict(
fs=self,
dictFromString=self.parseDict,
stringFromDict=self.marshalDict,
handleDict=lambda s, d: s.fs._handleRequest(s, d),
),
),
)
self.control_server.daemon_threads = True
# setup the threads last so that they don't prevent an exit.
self.control_thread = Thread(target=self.control_server.serve_forever, args=())
self.control_thread.start()
finally:
self.unlockGitFSDir()
mt = self.getMTab()
mt[mount_point] = self.getID()
self.updateMTab(mt)
self.repo = GitRepo(path, origin, branch, sync=True)
self.sync_thread = Thread(target=self._sync, args=())
self.sync_thread.start()
def getID(self):
if self.id is None:
self.id = mUUID.getUUIDFromFile(self.getUUIDFile(self.root), create=True).toString()
return self.id
def _lockWithTimeOut(self, name, t):
if t <= 0:
return
self.lock_lock.acquire()
expt = t + time()
self.locks[name] = expt
if self.lock_expire_time - expt < 0:
self.lock_expire_time = expt
if self.lock_timer != None:
self.lock_timer.cancel()
else:
logging.debug("Aquiring fresh lock")
self.sync_c.acquire()
self.lock_timer = Timer(t, self._lockTimerExpire, args=())
self.lock_timer.start()
self.lock_lock.release()
def _lockTimerExpire(self):
logging.debug("_lockTimeExpire")
self.lock_lock.acquire()
self.__lockTimerExpire()
self.lock_lock.release()
def __lockTimerExpire(self):
logging.debug("__lockTimeExpire")
now = time()
t = now
for key in self.locks:
if t - self.locks[key] < 0:
t = self.locks[key]
if now - self.locks[key] > 300:
del self.locks[key]
if t - now > 0:
self.lock_expire_time = t
if self.lock_timer != None:
self.lock_timer.cancel()
else:
logging.debug("***** ERROR ***** __lockTimerExpire doesn't have lock. acquiring")
self.sync_c.acquire()
self.lock_timer = Timer(t - now, self._lockTimerExpire, args=())
self.lock_timer.start()
logging.debug("extending lock.")
else:
if self.lock_timer != None:
logging.debug("releasing lock.")
self.lock_timer.cancel()
self.lock_timer = None
self.sync_c.release()
def _unlock(self, name):
if name not in self.locks:
return
self.lock_lock.acquire()
t = self.locks[name]
del self.locks[name]
if t >= self.lock_expire_time or len(keys(self.locks)) == 0:
self.__lockTimerExpire()
self.lock_lock.release()
def _handleRequest(self, request, d):
if d["action"] in self.handlers:
mf = self.handlers[d["action"]]
return mf(d, request)
logging.debug("No request in packet: %s" % d)
self._respond(request, {"status": "Unknown Command"})
return None
def _respond(self, request, responseDict):
request.sendDict(responseDict)
def _handlePing(self, reqDict, request):
self._respond(request, {"status": "ok", "message": "pong"})
def _handleLock(self, reqDict, request):
self._lockWithTimeOut("%s" % request.request.fileno(), 60)
self._respond(request, {"status": "ok", "name": "%s" % request.request.fileno()})
def _handleUnlock(self, reqDict, request):
self._unlock("%s" % request.request.fileno())
self._respond(request, {"status": "ok"})
def _handleInfo(self, reqDict, request):
self._respond(
request,
{
"status": "ok",
"origin": self.repo.origin,
"branch": self.repo.branch,
"root": self.root,
"path": self.mount_point,
},
)
def _getConfig(self, reqDict, request):
key = reqDict["key"]
resp = self.getConfigForInstance(key)
self._respond(request, {"status": "ok", key: resp})
def _sync(self):
while True:
self.sync_c.acquire()
if not self.halt:
# wait till a sync request comes
self.sync_c.wait()
self.timer_c.acquire()
if self.timer != None:
self.timer.cancel()
self.timer = None
self.timer_c.release()
try:
if not self.repo.synchronize():
# sync failed, we need to try again.
self.needSync()
self.need_sync_time = None
except Exception as e:
logging.debug("synchronize threw exception %s" % e)
self.sync_c.release() # can't release this until sync is complete because we can't change files while we sync.
else:
self.repo.forcePush()
self.repo.push()
self.sync_c.release()
break
def getHostInfo():
if self.hostinfo is None:
self.hostinfo = HostInfo()
self.hostinfo.update()
return self.hostinfo
def matchesInstance(self, key):
""" returns a number determinging how well the past in key matches the current instance."""
if key == "default":
return 0.1 # Very weak match
hostinfo = self.getHostInfo()
try:
ip = socket.inet_pton(key)
return hostinfo.matchAddress(ip)
except SocketError:
pass
return hostinfo.matchHostName(key)
def getConfigForInstanceSingleFile(self, key, name):
logging.debug("getConfigForInstanceSingleFile(%s, %s)" % (key, name))
c = self.getConfig(name)
if c is None:
return None
match = 0
if key not in c:
return None
v = c[key]
logging.debug("getConfigForInstanceSingleFile(%s, %s) value=%s" % (key, name, v))
if isinstance(v, dict):
rv = None
for (key, value) in v.iteritems():
tm = self.matchesInstance(key)
if tm > match:
match = tm
rv = value
v = rv
if v is None:
return None
if isinstance(v, list):
v = random.choose(v)
return (match, v)
def getConfigForInstance(self, key):
match = 0
value = None
logging.debug("getConfigForInstance(%s)" % key)
if key in self.config_file_priorities:
filenames = self.config_file_priorities[key]
else:
filenames = self.config_file_priorities["default"]
for name in filenames:
a = self.getConfigForInstanceSingleFile(key, name)
if a is None:
continue
(m, v) = a
if m >= match:
match = m
value = v
return value
def forceSync(self):
logging.debug("forceSync()")
self.sync_c.acquire()
self.sync_c.notifyAll()
self.sync_c.release()
def needSync(self):
logging.debug("needSync()")
self.timer_c.acquire()
if self.need_sync_time is None:
self.need_sync_time = time()
if self.timer != None:
if time() - self.need_sync_time > 5 * 60:
return
self.timer.cancel()
# don't do anything until there is a pause.
self.timer = Timer(10, self.forceSync, args=())
self.timer.start()
self.timer_c.release()
def shutdown(self):
# stop sync thread
self.sync_c.acquire()
self.halt = True
self.sync_c.notifyAll()
self.sync_c.release()
self.repo.shutDown()
def destroy(self, path):
self.shutdown()
if self.control_server != None:
self.control_server.shutdown()
try:
os.remove(self.control_socket_path)
except OSError:
pass
def __call__(self, op, path, *args):
try:
logging.debug("calling %s on %s" % (op, path))
path = self.escapePath(path)
r = super(GitFS, self).__call__(op, self.root + path, *args)
pr = "%s" % r
pr = pr[:10]
logging.debug("returning %s for %s" % (pr, op))
return r
except Exception as e:
logging.debug("Unhandled exception %s" % e)
raise e
def access(self, path, mode):
if not os.access(path, mode):
raise FuseOSError(EACCES)
def chmod(self, path, mode):
self.needSync()
return os.chmod(path, mode)
def chown(self, path, uid, gid):
self.needSync()
return super(GitFS, self).chown(path, uid, gid)
def create(self, path, mode):
# XXXXX Fixme what should the flags be?
return os.open(path, os.O_RDWR | os.O_CREAT, mode)
def flush(self, path, fh):
self.sync_c.acquire()
self.sync_c.notifyAll()
self.sync_c.release()
return os.fsync(fh)
def fsync(self, path, datasync, fh):
self.needSync()
return os.fsync(fh)
def fsyncdir(self, path, datasync, fh):
return self.fsync(path, datasync, fh)
def getattr(self, path, fh=None):
st = os.lstat(path)
return dict(
(key, getattr(st, key))
for key in ("st_atime", "st_ctime", "st_gid", "st_mode", "st_mtime", "st_nlink", "st_size", "st_uid")
)
getxattr = None
def link(self, target, source):
source = self.escapePath(source)
self.needSync()
return os.link(source, target)
listxattr = None
mknod = os.mknod
def mkdir(self, path, mode):
return os.mkdir(path, mode)
def open(self, path, fip):
f = os.open(path, fip)
logging.debug("open(%s, %s): %d" % (path, fip, f))
return f
def read(self, path, size, offset, fh):
with self.rwlock:
os.lseek(fh, offset, 0)
return os.read(fh, size)
def readdir(self, path, fh):
files = os.listdir(path)
uefiles = []
for file in files:
if self.isValidPath(file):
uefiles = uefiles + [self.unescapePath(file)]
return [".", ".."] + uefiles
readlink = os.readlink
def release(self, path, fh):
return os.close(fh)
def rename(self, old, new):
self.needSync()
return os.rename(old, self.root + self.escapePath(new))
def rmdir(self, path):
self.needSync()
return os.rmdir(path)
def statfs(self, path):
stv = os.statvfs(path)
return dict(
(key, getattr(stv, key))
for key in (
"f_bavail",
"f_bfree",
"f_blocks",
"f_bsize",
"f_favail",
"f_ffree",
"f_files",
"f_flag",
"f_frsize",
"f_namemax",
)
)
def symlink(self, target, source):
self.needSync()
return os.symlink(source, target)
def truncate(self, path, length, fh=None):
self.needSync()
with open(path, "r+") as f:
f.truncate(length)
def unlink(self, path):
self.needSync()
return os.unlink(path)
utimens = os.utime
def write(self, path, data, offset, fh):
self.needSync()
with self.rwlock:
os.lseek(fh, offset, 0)
return os.write(fh, data)
