def _propogateRename(self, item, change):
# TODO(conflicts): Verify inode numbers here
oldfusepath = item.getOldFilename()
newfusepath = item.getNewFilename()
os.rename(tsumufs.nfsPathOf(item.getOldFilename()),
tsumufs.nfsPathOf(item.getNewFilename()))
return False
def _propogateUnlink(self, item, change):
# TODO(conflicts): Conflict if the file type or inode have changed
fusepath = item.getFilename()
if item.getFileType() != 'dir':
os.unlink(tsumufs.nfsPathOf(fusepath))
else:
os.rmdir(tsumufs.nfsPathOf(fusepath))
return False
def ftruncate(self, size):
logger.debug('opcode: ftruncate | size: %d' % size)
try:
statgoo = tsumufs.cacheManager.statFile(self._path)
# Get inode number
try:
inum = tsumufs.NameToInodeMap.nameToInode(tsumufs.nfsPathOf(self._path))
except KeyError, e:
try:
inum = statgoo.st_ino
except (IOError, OSError), e:
inum = -1
except Exception, e:
exc_info = sys.exc_info()
logger.debug('*** Unhandled exception occurred')
logger.debug('*** Type: %s' % str(exc_info[0]))
logger.debug('*** Value: %s' % str(exc_info[1]))
logger.debug('*** Traceback:')
for line in traceback.extract_tb(exc_info[2]):
logger.debug('*** %s(%d) in %s: %s' % line)
def chmod(self, fusepath, mode):
'''
Chmod a file.
Returns:
None
Raises:
OSError, IOError
'''
self.lockFile(fusepath)
try:
opcodes = self._genCacheOpcodes(fusepath)
self._validateCache(fusepath, opcodes)
realpath = self._generatePath(fusepath, opcodes)
try:
perms = tsumufs.permsOverlay.getPerms(fusepath)
except KeyError:
statgoo = self.statFile(fusepath)
perms = { 'uid': statgoo.st_uid,
'gid': statgoo.st_gid,
'mode': statgoo.st_mode }
perms.mode = (perms.mode & 070000) | mode
tsumufs.permsOverlay.setPerms(fusepath, perms.uid, perms.gid, perms.mode)
self._invalidateStatCache(tsumufs.nfsPathOf(fusepath))
finally:
self.unlockFile(fusepath)
def _nfsDataChanged(self, fusepath):
'''
Check to see if the NFS data has changed since our last stat.
Returns:
Boolean true or false.
Raises:
Any error that might occur during an os.lstat(), aside from ENOENT.
'''
self.lockFile(fusepath)
try:
try:
cachedstat = self._cachedStats[fusepath]['stat']
realstat = os.lstat(tsumufs.nfsPathOf(fusepath))
if ((cachedstat.st_blocks != realstat.st_blocks) or
(cachedstat.st_mtime != realstat.st_mtime) or
(cachedstat.st_size != realstat.st_size) or
(cachedstat.st_ino != realstat.st_ino)):
return True
else:
return False
except OSError, e:
if e.errno == errno.ENOENT:
return False
else:
raise
except KeyError, e:
return False
def _generatePath(self, fusepath, opcodes=None):
'''
Return the path to use for all file operations, based upon the current state
of the world generated by _genCacheOpcodes.
Returns:
None
Raises:
Nothing
'''
if opcodes == None:
opcodes = self._genCacheOpcodes(fusepath)
logger.debug('Opcodes are: %s' % opcodes)
for opcode in opcodes:
if opcode == 'enoent':
logger.debug('ENOENT on %s' % fusepath)
raise OSError(errno.ENOENT, os.strerror(errno.ENOENT))
if opcode == 'use-nfs':
logger.debug('Returning nfs path for %s' % fusepath)
return tsumufs.nfsPathOf(fusepath)
if opcode == 'use-cache':
logger.debug('Returning cache path for %s' % fusepath)
return tsumufs.cachePathOf(fusepath)
def truncateFile(self, fusepath, newsize):
'''
Truncate a file to newsize.
'''
try:
self.lockFile(fusepath)
try:
nfspath = tsumufs.nfsPathOf(fusepath)
fp = open(nfspath, 'r+')
fp.truncate(newsize)
fp.close()
except OSError, e:
if e.errno in (errno.EIO, errno.ESTALE):
logger.debug('Got %s while writing a region to %s.' %
(str(e), nfspath))
logger.debug('Triggering a disconnect.')
tsumufs.nfsAvailable.clear()
tsumufs.nfsAvailable.notifyAll()
raise tsumufs.NFSMountError()
else:
raise
finally:
self.unlockFile(fusepath)
def _propogateNew(self, item, change):
fusepath = item.getFilename()
nfspath = tsumufs.nfsPathOf(fusepath)
# Horrible hack, but it works to test for the existance of a file.
try:
tsumufs.nfsMount.readFileRegion(fusepath, 0, 0)
except (OSError, IOError), e:
if e.errno != errno.ENOENT:
return True
def _cacheDir(self, fusepath):
'''
Cache the directory referenced by path.
If the directory should not be cached to disk (as specified in the
cachespec) then only the contents of the directory hash table will
be stored in the _cachedFiles hash.
Returns:
None
Raises:
OSError - when an error operating on the filesystem occurs.
'''
self.lockFile(fusepath)
try:
nfspath = tsumufs.nfsPathOf(fusepath)
cachepath = tsumufs.cachePathOf(fusepath)
stat = os.lstat(nfspath)
logger.debug('nfspath = %s' % nfspath)
logger.debug('cachepath = %s' % cachepath)
if fusepath == '/':
logger.debug('Asking to cache root -- skipping the cache to '
'disk operation, but caching data in memory.')
else:
try:
os.mkdir(cachepath)
except OSError, e:
# Skip EEXIST errors -- if it already exists, it may have files in it
# already. Simply copy the stat and chown it again, then cache the
# listdir operation as well.
if e.errno != errno.EEXIST:
raise
shutil.copystat(nfspath, cachepath)
tsumufs.permsOverlay.setPerms(fusepath,
stat.st_uid,
stat.st_gid,
stat.st_mode)
logger.debug('Caching directory %s to disk.' % fusepath)
self._cachedDirents[fusepath] = os.listdir(nfspath)
def writeFileRegion(self, filename, start, end, data):
'''
Method to write a region to a file on the NFS mount. Additionally
adds the resulting inode to filename mapping to the InodeMap
singleton.
Args:
filename: the complete pathname to the file to write to.
start: the beginning offset to write to.
end: the ending offset to write to.
data: the data to write.
Raises:
NFSMountError: An error occurred during an NFS call.
RangeError: The start and end provided are invalid.
OSError: Usually relating to permissions on the file.
'''
if end - start > len(data):
raise dataregion.RangeError('The length of data specified in start and '
'end does not match the data length.')
try:
self.lockFile(filename)
try:
nfspath = tsumufs.nfsPathOf(filename)
fp = open(nfspath, 'r+')
fp.seek(start)
fp.write(data)
fp.close()
except OSError, e:
if e.errno in (errno.EIO, errno.ESTALE):
logger.debug('Got %s while writing a region to %s.' %
(str(e), filename))
logger.debug('Triggering a disconnect.')
tsumufs.nfsAvailable.clear()
tsumufs.nfsAvailable.notifyAll() # TODO: AttributeError
raise tsumufs.NFSMountError()
else:
raise
finally:
self.unlockFile(filename)
def readFileRegion(self, filename, start, end):
'''
Method to read a region of a file from the NFS mount. Additionally
adds the inode to filename mapping to the InodeMap singleton.
Args:
filename: the complete pathname to the file to read from.
start: the beginning offset to read from.
end: the ending offset to read from.
Returns:
A string containing the data read.
Raises:
NFSMountError: An error occurred during an NFS call which is
unrecoverable.
RangeError: The start and end provided are invalid.
IOError: Usually relating to permissions issues on the file.
'''
try:
self.lockFile(filename)
try:
nfspath = tsumufs.nfsPathOf(filename)
fp = open(nfspath, 'r')
fp.seek(start)
result = fp.read(end - start)
fp.close()
return result
except OSError, e:
if e.errno in (errno.EIO, errno.ESTALE):
logger.debug('Got %s while reading a region from %s.' %
(str(e), filename))
logger.debug('Triggering a disconnect.')
tsumufs.nfsAvailable.clear()
tsumufs.nfsAvailable.notifyAll()
raise tsumufs.NFSMountError()
else:
raise
finally:
self.unlockFile(filename)
def write(self, new_data, offset):
logger.debug('opcode: write | path: %s | offset: %d | buf: %s'
% (self._path, offset, repr(new_data)))
# Three cases here:
# - The file didn't exist prior to our write.
# - The file existed, but was extended.
# - The file existed, and an existing block was overwritten.
nfspath = tsumufs.nfsPathOf(self._path)
statgoo = tsumufs.cacheManager.statFile(self._path)
try:
inode = tsumufs.NameToInodeMap.nameToInode(nfspath)
except KeyError, e:
try:
inode = statgoo.st_ino
except (IOError, OSError), e:
inode = -1
def _cacheFile(self, fusepath):
'''
Cache the file referenced by path.
This method locks the file for reading, determines what type it
is, and attempts to cache it. Note that if there was an issue
reading from the NFSMount, this method will mark the NFS mount as
being unavailble.
Note: The touch cache isn't implemented here at the moment. As a
result, the entire cache is considered permacache for now.
Note: NFS error checking and disable are not handled here for the
moment. Any errors that would ordinarily shut down the NFS mount
are just reported as normal OSErrors, aside from ENOENT.
Returns:
Nothing.
Raises:
OSError if there was an issue attempting to copy the file
across to cache.
'''
# TODO(jtg): Add support for storing the UID/GID
self.lockFile(fusepath)
try:
logger.debug('Caching file %s to disk.' % fusepath)
nfspath = tsumufs.nfsPathOf(fusepath)
cachepath = tsumufs.cachePathOf(fusepath)
curstat = os.lstat(nfspath)
if (stat.S_ISREG(curstat.st_mode) or
stat.S_ISFIFO(curstat.st_mode) or
stat.S_ISSOCK(curstat.st_mode) or
stat.S_ISCHR(curstat.st_mode) or
stat.S_ISBLK(curstat.st_mode)):
shutil.copy(nfspath, cachepath)
shutil.copystat(nfspath, cachepath)
elif stat.S_ISLNK(curstat.st_mode):
dest = os.readlink(nfspath)
try:
os.unlink(cachepath)
except OSError, e:
if e.errno != errno.ENOENT:
raise
os.symlink(dest, cachepath)
#os.lchown(cachepath, curstat.st_uid, curstat.st_gid)
#os.lutimes(cachepath, (curstat.st_atime, curstat.st_mtime))
elif stat.S_ISDIR(curstat.st_mode):
# Caching a directory to disk -- call cacheDir instead.
logger.debug('Request to cache a directory -- calling _cacheDir')
self._cacheDir(fusepath)
def _propogateChange(self, item, change):
# Rules:
# 1. On conflict NFS always wins.
# 2. Loser data always appended as a list of changes in
# ${mount}/._${file}.changes
# 3. We're no better than NFS
# Steps:
# 1. Stat both files, and verify the file type is identical.
# 2. Read in the regions from NFS.
# 3. Compare the regions between the changes and NFS.
# 4. If any changes differ, the entire set is conflicted.
# 4a. Create a conflict change file and write out the changes
# that differ.
# 4b. Create a 'new' change in the synclog for the conflict
# change file.
# 4c. Erase the cached file on disk.
# 4d. Invalidate dirent cache for the file's containing dir.
# 4e. Invalidate the stat cache fo that file.
# 5. Otherwise:
# 4a. Iterate over each change and write it out to NFS.
fusepath = item.getFilename()
logger.debug('Fuse path is %s' % fusepath)
nfs_stat = os.lstat(tsumufs.nfsPathOf(fusepath))
cache_stat = os.lstat(tsumufs.cachePathOf(fusepath))
logger.debug('Validating data hasn\'t changed on NFS.')
if stat.S_IFMT(nfs_stat.st_mode) != stat.S_IFMT(cache_stat.st_mode):
logger.debug('File type has completely changed -- conflicted.')
return True
elif nfs_stat.st_ino != item.getInum():
logger.debug('Inode number changed -- conflicted.')
return True
else:
# Iterate over each region, and verify the changes
for region in change.getDataChanges():
data = tsumufs.nfsMount.readFileRegion(fusepath,
region.getStart(),
region.getEnd()-region.getStart())
if len(data) < region.getEnd() - region.getStart():
data += '\x00' * ((region.getEnd() - region.getStart()) - len(data))
if region.getData() != data:
logger.debug('Region has changed -- entire changeset conflicted.')
logger.debug('Data read was %s' % repr(data))
logger.debug('Wanted %s' % repr(region.getData()))
return True
logger.debug('No conflicts detected.')
# Propogate changes
for region in change.getDataChanges():
data = tsumufs.cacheManager.readFile(fusepath,
region.getStart(),
region.getEnd()-region.getStart(),
os.O_RDONLY)
# Pad the region with nulls if we get a short read (EOF before the end of
# the real file. It means we ran into a truncate issue and that the file
# is shorter than it was originally -- we'll propogate the truncate down
# the line.
if len(data) < region.getEnd() - region.getStart():
data += '\x00' * ((region.getEnd() - region.getStart()) - len(data))
logger.debug('Writing to %s at [%d-%d] %s'
% (fusepath, region.getStart(),
region.getEnd(), repr(data)))
tsumufs.nfsMount.writeFileRegion(fusepath,
region.getStart(),
region.getEnd(),
data)
# Propogate truncations
if (cache_stat.st_size < nfs_stat.st_size):
tsumufs.nfsMount.truncateFile(fusepath, cache_stat.st_size)
# TODO(writeback): Add in metadata syncing here.
return False
