def _generatePath(self, fusepath, opcodes=None): ''' Return the path to use for all file operations, based upon the current state of the world generated by _genCacheOpcodes. Returns: None Raises: Nothing ''' if opcodes == None: opcodes = self._genCacheOpcodes(fusepath) logger.debug('Opcodes are: %s' % opcodes) for opcode in opcodes: if opcode == 'enoent': logger.debug('ENOENT on %s' % fusepath) raise OSError(errno.ENOENT, os.strerror(errno.ENOENT)) if opcode == 'use-nfs': logger.debug('Returning nfs path for %s' % fusepath) return tsumufs.nfsPathOf(fusepath) if opcode == 'use-cache': logger.debug('Returning cache path for %s' % fusepath) return tsumufs.cachePathOf(fusepath)
def isFileDirty(self, fusepath, recursive=False): ''' Check to see if the cached copy of a file is dirty. Note that this does a shortcut test -- if the file in local cache exists and the file on fs does not, then we assume the cached copy is dirty. Otherwise, we have to check against the synclog to see what's changed (if at all). Returns: Boolean true or false. Raises: Any error that might occur during an os.lstat(), aside from ENOENT. ''' try: self._lock.acquire() self.getChange(filename=fusepath, pk=True) return True except tsumufs.DocumentException, e: if recursive and os.path.isdir(tsumufs.cachePathOf(fusepath)): try: self._syncChanges.by_dir_prefix(key=fusepath, pk=True) return True except tsumufs.DocumentException, e: pass
def isCachedToDisk(self, fusepath): ''' Check to see if the file referenced by fusepath is cached to disk. Fusepath is expected to be an absolute path into the filesystem from the view seen from FUSE. Ie: all "absolute paths" are actually relative to the tsumufs mountpoint root. Returns: Boolean. True if the file is cached. False otherwise. Raises: OSError if there was an issue statting the file in question. ''' # Lock the file for access self.lockFile(fusepath) try: try: statgoo = os.lstat(tsumufs.cachePathOf(fusepath)) if stat.S_ISDIR(statgoo.st_mode) and tsumufs.nfsAvailable.isSet(): return self._cachedDirents.has_key(fusepath) except OSError, e: if e.errno == errno.ENOENT: return False else: logger.debug('_isCachedToDisk: Caught OSError: errno %d: %s' % (e.errno, e.strerror)) raise else:
def isCachedToDisk(self, fusepath): ''' Check to see if the file referenced by fusepath is cached to disk. Fusepath is expected to be an absolute path into the filesystem from the view seen from FUSE. Ie: all "absolute paths" are actually relative to the tsumufs mountpoint root. Returns: Boolean: True if the file is cached. False otherwise. Raises: OSError if there was an issue statting the file in question. ''' # Lock the file for access self.lockFile(fusepath) try: try: # TODO: check the file in the cached revisions, # instead of stating the fs. statgoo = os.lstat(tsumufs.cachePathOf(fusepath)) except OSError, e: if e.errno == errno.ENOENT: return False else: self._debug('_isCachedToDisk: Caught OSError: errno %d: %s' % (e.errno, e.strerror)) raise return True
def makeDir(self, fusepath): self.lockFile(fusepath) try: opcodes = self._genCacheOpcodes(fusepath) # Skip enoents -- we're creating a dir. try: self._validateCache(fusepath, opcodes) except (IOError, OSError), e: if e.errno != errno.ENOENT: raise realpath = tsumufs.cachePathOf(fusepath) dirname = os.path.dirname(fusepath) basename = os.path.basename(fusepath) if 'use-cache' in opcodes: if self._cachedDirents.has_key(dirname): self._cachedDirents[dirname].append(basename) self._cachedDirents[fusepath] = [] self._invalidateStatCache(realpath) logger.debug("Making directory %s" % realpath) return os.mkdir(realpath, 0755)
def writeFile(self, fusepath, offset, buf, flags, mode=None): ''' Write a chunk of data to the file referred to by fusepath. This method acts very much like the typical idiom: fp = open(file, mode) fp.seek(offset) result = fp.write(buf) return result Except that all writes go diractly to the cache first, and a synclog entry is created. Returns: The number of bytes written. Raises: OSError on error writing the data. IOError on error writing the data. ''' self.lockFile(fusepath) try: opcodes = self._genCacheOpcodes(fusepath) self._validateCache(fusepath, opcodes) realpath = tsumufs.cachePathOf(fusepath) logger.debug('Writing to file %s at offset %d with buffer length of %d ' 'and mode %s' % (realpath, offset, len(buf), mode)) # TODO(jtg): Validate permissions here, too if mode != None: fd = os.open(realpath, flags, mode) else: fd = os.open(realpath, flags) fp = os.fdopen(fd, self._flagsToStdioMode(flags)) if offset >= 0: fp.seek(offset) else: fp.seek(0, 2) bytes_written = fp.write(buf) fp.close() # Since we wrote to the file, invalidate the stat cache if it exists. self._invalidateStatCache(realpath) return bytes_written finally: self.unlockFile(fusepath)
def _cacheDir(self, fusepath): ''' Cache the directory referenced by path. If the directory should not be cached to disk (as specified in the cachespec) then only the contents of the directory hash table will be stored in the _cachedFiles hash. Returns: None Raises: OSError - when an error operating on the filesystem occurs. ''' self.lockFile(fusepath) try: fspath = tsumufs.fsPathOf(fusepath) cachepath = tsumufs.cachePathOf(fusepath) self._debug('fspath = %s' % fspath) self._debug('cachepath = %s' % cachepath) if fusepath == '/': self._debug('Asking to cache root -- skipping the cache to ' 'disk operation, but caching data in memory.') else: try: os.mkdir(cachepath) except OSError, e: # Skip EEXIST errors -- if it already exists, it may have files in it # already. Simply copy the stat and chown it again, then cache the # listdir operation as well. if e.errno != errno.EEXIST: raise # If activated, populating database will compare the filesystem dirents # to the database dirents, and add to database any files that could be # created bypassing the CouchedFilesystem api, directly on the filesystem. # The performance overhead raised by this operation increase when the number # of file to add to database is higher. if tsumufs.populateDb: try: self._debug('Discovering directory %s contents and populate database.' % fusepath) dirents = [ doc.filename for doc in tsumufs.fsOverlay.listdir(fusepath) ] for filename in os.listdir(fspath): if filename not in dirents: tsumufs.fsOverlay.populate(os.path.join(fusepath, filename)) except OSError, e: self._debug('Cannot list directory %s (%s)' % (fusepath, e.strerror))
def getDirents(self, fusepath): ''' Return the dirents from a directory's contents if cached. ''' self.lockFile(fusepath) try: opcodes = self._genCacheOpcodes(fusepath) self._validateCache(fusepath, opcodes) if 'enoent' in opcodes: raise OSError(errno.ENOENT, os.strerror(errno.ENOENT)) if tsumufs.nfsAvailable.isSet(): logger.debug('NFS is available -- combined dirents from NFS and ' 'cached disk.') nfs_dirents = set(self._cachedDirents[fusepath]) cached_dirents = set(os.listdir(tsumufs.cachePathOf(fusepath))) final_dirents_list = [ '.', '..' ] for dirent in nfs_dirents.union(cached_dirents): final_dirents_list.append(dirent) logger.debug('nfs_dirents = %s' % nfs_dirents); logger.debug('cached_dirents = %s' % cached_dirents); logger.debug('final_dirents_list = %s' % final_dirents_list); return final_dirents_list else: logger.debug('NFS is unavailable -- returning cached disk dir stuff.') dirents = [ '.', '..' ] dirents.extend(os.listdir(tsumufs.cachePathOf(fusepath))) return dirents finally: self.unlockFile(fusepath)
def _propagateNew(self, item, change): fusepath = item.filename cachepath = tsumufs.cachePathOf(fusepath) try: document = tsumufs.fsOverlay[fusepath] os.lstat(cachepath) except OSError, e: if e.errno == errno.ENOENT: # The file may have been deleted while we # were waiting for the lock return False
def _getFileInum(self, fusepath): ''' Return the inode number of the file specified in the cache. Returns: The inode number. Raises: OSError ''' cachepath = tsumufs.cachePathOf(fusepath) inum = os.lstat(cachepath).st_ino return inum
def _cacheDir(self, fusepath): ''' Cache the directory referenced by path. If the directory should not be cached to disk (as specified in the cachespec) then only the contents of the directory hash table will be stored in the _cachedFiles hash. Returns: None Raises: OSError - when an error operating on the filesystem occurs. ''' self.lockFile(fusepath) try: nfspath = tsumufs.nfsPathOf(fusepath) cachepath = tsumufs.cachePathOf(fusepath) stat = os.lstat(nfspath) logger.debug('nfspath = %s' % nfspath) logger.debug('cachepath = %s' % cachepath) if fusepath == '/': logger.debug('Asking to cache root -- skipping the cache to ' 'disk operation, but caching data in memory.') else: try: os.mkdir(cachepath) except OSError, e: # Skip EEXIST errors -- if it already exists, it may have files in it # already. Simply copy the stat and chown it again, then cache the # listdir operation as well. if e.errno != errno.EEXIST: raise shutil.copystat(nfspath, cachepath) tsumufs.permsOverlay.setPerms(fusepath, stat.st_uid, stat.st_gid, stat.st_mode) logger.debug('Caching directory %s to disk.' % fusepath) self._cachedDirents[fusepath] = os.listdir(nfspath)
def removeCachedFile(self, fusepath): ''' Remove the cached file referenced by fusepath from the cache. This method locks the file, determines what type it is, and attempts to decache it. Note: The touch cache isn't implemented here at the moment. As a result, the entire cache is considered permacache for now. Returns: None Raises: OSError if there was an issue attempting to remove the file from cache. ''' self.lockFile(fusepath) try: cachefilename = tsumufs.cachePathOf(fusepath) ino = os.lstat(cachefilename).st_ino if os.path.isfile(cachefilename) or os.path.islink(cachefilename): os.unlink(cachefilename) elif os.path.isdir(cachefilename): os.rmdir(cachefilename) # Invalidate the stat cache for this file self._invalidateStatCache(cachefilename) # Remove this file from the dirent cache if it was put in there. self._invalidateDirentCache(os.path.dirname(fusepath), os.path.basename(fusepath)) # Remove this file from the permsOverlay tsumufs.permsOverlay.removePerms(ino) finally: self.unlockFile(fusepath)
def _cacheFile(self, fusepath): ''' Cache the file referenced by path. This method locks the file for reading, determines what type it is, and attempts to cache it. Note that if there was an issue reading from the NFSMount, this method will mark the NFS mount as being unavailble. Note: The touch cache isn't implemented here at the moment. As a result, the entire cache is considered permacache for now. Note: NFS error checking and disable are not handled here for the moment. Any errors that would ordinarily shut down the NFS mount are just reported as normal OSErrors, aside from ENOENT. Returns: Nothing. Raises: OSError if there was an issue attempting to copy the file across to cache. ''' # TODO(jtg): Add support for storing the UID/GID self.lockFile(fusepath) try: logger.debug('Caching file %s to disk.' % fusepath) nfspath = tsumufs.nfsPathOf(fusepath) cachepath = tsumufs.cachePathOf(fusepath) curstat = os.lstat(nfspath) if (stat.S_ISREG(curstat.st_mode) or stat.S_ISFIFO(curstat.st_mode) or stat.S_ISSOCK(curstat.st_mode) or stat.S_ISCHR(curstat.st_mode) or stat.S_ISBLK(curstat.st_mode)): shutil.copy(nfspath, cachepath) shutil.copystat(nfspath, cachepath) elif stat.S_ISLNK(curstat.st_mode): dest = os.readlink(nfspath) try: os.unlink(cachepath) except OSError, e: if e.errno != errno.ENOENT: raise os.symlink(dest, cachepath) #os.lchown(cachepath, curstat.st_uid, curstat.st_gid) #os.lutimes(cachepath, (curstat.st_atime, curstat.st_mtime)) elif stat.S_ISDIR(curstat.st_mode): # Caching a directory to disk -- call cacheDir instead. logger.debug('Request to cache a directory -- calling _cacheDir') self._cacheDir(fusepath)
def _writeChangeSet(self, item, change): # TODO(refactor): Make FileChange generate the patch set string instead. if item.getType() != 'rename': fusepath = item.getFilename() else: fusepath = item.getOldFilename() if fusepath[0] == '/': conflictpath = fusepath[1:] else: conflictpath = fusepath conflictpath = conflictpath.replace('/', '-') conflictpath = os.path.join(tsumufs.conflictDir, conflictpath) logger.debug('Using %s as the conflictpath.' % conflictpath) try: tsumufs.cacheManager.lockFile(fusepath) isNewFile = True fd = None try: logger.debug('Attempting open of %s' % conflictpath) tsumufs.cacheManager.fakeOpen(conflictpath, os.O_CREAT|os.O_APPEND|os.O_RDWR, 0700 | stat.S_IFREG); fd = os.open(tsumufs.cachePathOf(conflictpath), os.O_CREAT|os.O_APPEND|os.O_RDWR, 0700 | stat.S_IFREG) isNewFile = True except OSError, e: if e.errno != errno.EEXIST: raise isNewFile = False logger.debug('File existed -- reopening as O_APPEND' % conflictpath) tsumufs.cacheManager.fakeOpen(conflictpath, os.O_APPEND|os.O_RDWR|os.O_EXCL, 0700 | stat.S_IFREG); fd = os.open(tsumufs.cachePathOf(conflictpath), os.O_APPEND|os.O_RDWR|os.O_EXCL, 0700 | stat.S_IFREG) fp = os.fdopen(fd, 'r+') startPos = fp.tell() fp.close() # Write the changeset preamble logger.debug('Writing preamble.') tsumufs.cacheManager.writeFile(conflictpath, -1, CONFLICT_PREAMBLE % { 'timestamp': time.time() }, os.O_APPEND|os.O_RDWR) if item.getType() == 'new': # TODO(conflicts): Write the entire file to the changeset as one large # patch. logger.debug('New file -- don\'t know what to do -- skipping.') pass if item.getType() == 'change': # TODO(conflicts): Propogate metadata changes as well. # TODO(conflicts): Propogate truncates! # Write changes to file logger.debug('Writing changes to conflict file.') for region in change.getDataChanges(): data = tsumufs.cacheManager.readFile(fusepath, region.getStart(), region.getEnd()-region.getStart(), os.O_RDONLY) tsumufs.cacheManager.writeFile(conflictpath, -1, 'set.addChange(type_="patch", start=%d, end=%d, data=%s)' % (region.getStart(), region.getEnd(), repr(data)), os.O_APPEND|os.O_RDWR) if item.getType() == 'link': # TODO(conflicts): Implement links. logger.debug('Link file -- don\'t know what to do -- skipping.') pass if item.getType() == 'unlink': fp.write('set.addUnlink()') if item.getType() == 'symlink': # TODO(conflicts): Implement symlinks. logger.debug('Symlink file -- don\'t know what to do -- skipping.') pass if item.getType() == 'rename': logger.debug('Rename file -- don\'t know what to do -- skipping.') pass logger.debug('Writing postamble.') tsumufs.cacheManager.writeFile(conflictpath, -1, CONFLICT_POSTAMBLE, os.O_APPEND|os.O_RDWR) logger.debug('Getting file size.') fp = open(tsumufs.cachePathOf(conflictpath), 'r+') fp.seek(0, 2) endPos = fp.tell() fp.close() if isNewFile: logger.debug('Conflictfile was new -- adding to synclog.') tsumufs.syncLog.addNew('file', filename=conflictpath) perms = tsumufs.cacheManager.statFile(fusepath) tsumufs.permsOverlay.setPerms(conflictpath, perms.st_uid, perms.st_gid, 0700 | stat.S_IFREG) logger.debug('Setting permissions to (%d, %d, %o)' % (perms.st_uid, perms.st_gid, 0700 | stat.S_IFREG)) else: logger.debug('Conflictfile was preexisting -- adding change.') tsumufs.syncLog.addChange(conflictpath, -1, startPos, endPos, '\x00' * (endPos - startPos))
def _propogateChange(self, item, change): # Rules: # 1. On conflict NFS always wins. # 2. Loser data always appended as a list of changes in # ${mount}/._${file}.changes # 3. We're no better than NFS # Steps: # 1. Stat both files, and verify the file type is identical. # 2. Read in the regions from NFS. # 3. Compare the regions between the changes and NFS. # 4. If any changes differ, the entire set is conflicted. # 4a. Create a conflict change file and write out the changes # that differ. # 4b. Create a 'new' change in the synclog for the conflict # change file. # 4c. Erase the cached file on disk. # 4d. Invalidate dirent cache for the file's containing dir. # 4e. Invalidate the stat cache fo that file. # 5. Otherwise: # 4a. Iterate over each change and write it out to NFS. fusepath = item.getFilename() logger.debug('Fuse path is %s' % fusepath) nfs_stat = os.lstat(tsumufs.nfsPathOf(fusepath)) cache_stat = os.lstat(tsumufs.cachePathOf(fusepath)) logger.debug('Validating data hasn\'t changed on NFS.') if stat.S_IFMT(nfs_stat.st_mode) != stat.S_IFMT(cache_stat.st_mode): logger.debug('File type has completely changed -- conflicted.') return True elif nfs_stat.st_ino != item.getInum(): logger.debug('Inode number changed -- conflicted.') return True else: # Iterate over each region, and verify the changes for region in change.getDataChanges(): data = tsumufs.nfsMount.readFileRegion(fusepath, region.getStart(), region.getEnd()-region.getStart()) if len(data) < region.getEnd() - region.getStart(): data += '\x00' * ((region.getEnd() - region.getStart()) - len(data)) if region.getData() != data: logger.debug('Region has changed -- entire changeset conflicted.') logger.debug('Data read was %s' % repr(data)) logger.debug('Wanted %s' % repr(region.getData())) return True logger.debug('No conflicts detected.') # Propogate changes for region in change.getDataChanges(): data = tsumufs.cacheManager.readFile(fusepath, region.getStart(), region.getEnd()-region.getStart(), os.O_RDONLY) # Pad the region with nulls if we get a short read (EOF before the end of # the real file. It means we ran into a truncate issue and that the file # is shorter than it was originally -- we'll propogate the truncate down # the line. if len(data) < region.getEnd() - region.getStart(): data += '\x00' * ((region.getEnd() - region.getStart()) - len(data)) logger.debug('Writing to %s at [%d-%d] %s' % (fusepath, region.getStart(), region.getEnd(), repr(data))) tsumufs.nfsMount.writeFileRegion(fusepath, region.getStart(), region.getEnd(), data) # Propogate truncations if (cache_stat.st_size < nfs_stat.st_size): tsumufs.nfsMount.truncateFile(fusepath, cache_stat.st_size) # TODO(writeback): Add in metadata syncing here. return False
def _writeChangeSet(self, item, change): # TODO(refactor): Make SyncItem generate the patch set string instead. if item.type != 'rename': fusepath = item.filename else: fusepath = item.old_fname if fusepath[0] == '/': conflictpath = fusepath[1:] else: conflictpath = fusepath conflictpath = conflictpath.replace('/', '-') conflictpath = os.path.join(tsumufs.conflictDir, conflictpath) self._debug('Using %s as the conflictpath.' % conflictpath) try: tsumufs.cacheManager.lockFile(fusepath) isNewFile = True fd = None try: self._debug('Attempting open of %s' % conflictpath) # To do open files in binary mode on Windows (os.O_BINARY) tsumufs.cacheManager.fakeOpen(conflictpath, os.O_CREAT|os.O_APPEND|os.O_RDWR, 0700 | stat.S_IFREG); fd = os.open(tsumufs.cachePathOf(conflictpath), os.O_CREAT|os.O_APPEND|os.O_RDWR, 0700 | stat.S_IFREG) isNewFile = True except OSError, e: if e.errno != errno.EEXIST: raise isNewFile = False self._debug('File existed -- reopening as O_APPEND' % conflictpath) tsumufs.cacheManager.fakeOpen(conflictpath, os.O_APPEND|os.O_RDWR|os.O_EXCL, 0700 | stat.S_IFREG); fd = os.open(tsumufs.cachePathOf(conflictpath), os.O_APPEND|os.O_RDWR|os.O_EXCL, 0700 | stat.S_IFREG) fp = os.fdopen(fd, 'r+') startPos = fp.tell() fp.close() # Write the changeset preamble self._debug('Writing preamble.') tsumufs.cacheManager.writeFile(conflictpath, -1, CONFLICT_PREAMBLE % { 'timestamp': time.time() }, os.O_APPEND|os.O_RDWR) if item.type == 'new': # TODO(conflicts): Write the entire file to the changeset as one large # patch. self._debug('New file -- don\'t know what to do -- skipping.') pass if item.type == 'change': # TODO(conflicts): propagate metadata changes as well. # TODO(conflicts): propagate truncates! # Write changes to file self._debug('Writing changes to conflict file.') for region in change.getDataChanges(): data = tsumufs.cacheManager.readFile(fusepath, region.start, region.end - region.start, os.O_RDONLY) tsumufs.cacheManager.writeFile(conflictpath, -1, 'set.addChange(type_="patch", start=%d, end=%d, data=%s)' % (region.start, region.end, repr(data)), os.O_APPEND|os.O_RDWR) if item.type == 'link': # TODO(conflicts): Implement links. self._debug('Link file -- don\'t know what to do -- skipping.') pass if item.type == 'unlink': fp.write('set.addUnlink()') if item.type == 'symlink': # TODO(conflicts): Implement symlinks. self._debug('Symlink file -- don\'t know what to do -- skipping.') pass if item.type == 'rename': self._debug('Rename file -- don\'t know what to do -- skipping.') pass self._debug('Writing postamble.') tsumufs.cacheManager.writeFile(conflictpath, -1, CONFLICT_POSTAMBLE, os.O_APPEND|os.O_RDWR) self._debug('Getting file size.') fp = open(tsumufs.cachePathOf(conflictpath), 'r+') fp.seek(0, 2) endPos = fp.tell() fp.close() if isNewFile: self._debug('Conflictfile was new -- adding to synclog.') tsumufs.syncLog.addNew('file', filename=conflictpath) # document = tsumufs.cacheManager.statFile(fusepath) # tsumufs.fsOverlay.open(conflictpath, create=True, # uid=document.uid, gid=document.gid, # mode=0700 | stat.S_IFREG) else: self._debug('Conflictfile was preexisting -- adding change.') tsumufs.syncLog.addChange(conflictpath, startPos, endPos, '\x00' * (endPos - startPos))
def _cacheFile(self, fusepath): ''' Cache the file referenced by path. This method locks the file for reading, determines what type it is, and attempts to cache it. Note that if there was an issue reading from the fsMount, this method will mark the fs mount as being unavailble. Note: The touch cache isn't implemented here at the moment. As a result, the entire cache is considered permacache for now. Note: fs error checking and disable are not handled here for the moment. Any errors that would ordinarily shut down the fs mount are just reported as normal OSErrors, aside from ENOENT. Returns: Nothing. Raises: OSError if there was an issue attempting to copy the file across to cache. ''' # TODO(jtg): Add support for storing the UID/GID self.lockFile(fusepath) try: fspath = tsumufs.fsPathOf(fusepath) cachepath = tsumufs.cachePathOf(fusepath) document = tsumufs.fsOverlay[fusepath] if stat.S_ISDIR(document.mode): # Caching a directory to disk -- call cacheDir instead. self._debug('Request to cache a directory -- calling _cacheDir') self._cacheDir(fusepath) else: self._debug('Caching file %s to disk.' % fusepath) if (stat.S_ISREG(document.mode) or stat.S_ISFIFO(document.mode) or stat.S_ISSOCK(document.mode) or stat.S_ISCHR(document.mode) or stat.S_ISBLK(document.mode)): try: shutil.copyfileobj(tsumufs.fsMount.open(fusepath, os.O_RDONLY | os.O_BINARY), open(cachepath, "wb")) except AttributeError, e: shutil.copyfileobj(tsumufs.fsMount.open(fusepath, os.O_RDONLY), open(cachepath, "w")) elif stat.S_ISLNK(document.mode): dest = os.readlink(fspath) try: os.unlink(cachepath) except OSError, e: if e.errno != errno.ENOENT: raise os.symlink(dest, cachepath)