def read(self, path, size=0, offset=0, id=None): """ Read the required bytes from the file and return a buffer containing the bytes. """ ## Read from the requested filehandle, which was set during 'open' if id is None: raise FuseOSError(EIO) vos.logger.debug("reading range: %s %d %d %d" % (path, size, offset, id)) try: fh = HandleWrapper.findHandle(id) except KeyError: raise FuseOSError(EIO) try: return fh.cacheFileHandle.read(size, offset) except CacheRetry: e = FuseOSError(EAGAIN) e.strerror = "Timeout waiting for file read" vos.logger.debug("Timeout Waiting for file read: %s", path) raise e
def create(self, path, flags): """Create a node. Currently ignores the ownership mode""" vos.logger.debug("Creating a node: %s with flags %s" % (path, str(flags))) # Create is handle by the client. # This should fail if the basepath doesn't exist try: self.client.open(path, os.O_CREAT).close() node = self.getNode(path) parent = self.getNode(os.path.dirname(path)) # Force inheritance of group settings. node.groupread = parent.groupread node.groupwrite = parent.groupwrite if node.chmod(flags): # chmod returns True if the mode changed but doesn't do update. self.client.update(node) node = self.getNode(path, force=True) except Exception as e: vos.logger.error(str(e)) vos.logger.error("Error trying to create Node %s" % (path)) f = FuseOSError(getattr(e, 'errno', EIO)) f.strerror = getattr(e, 'strerror', 'failed to create %s' % (path)) raise f ## now we can just open the file in the usual way and return the handle return self.open(path, os.O_WRONLY)
def write(self, path, data, size, offset, id=None): import ctypes if self.opt.readonly: vos.logger.debug("File system is readonly.. so writing 0 bytes\n") return 0 try: fh = HandleWrapper.findHandle(id) except KeyError: raise FuseOSError(EIO) if fh.readOnly: vos.logger.debug("file was not opened for writing") raise FuseOSError(EPERM) vos.logger.debug("%s -> %s" % (path, fh)) vos.logger.debug("%d --> %d" % (offset, offset + size)) try: return fh.cacheFileHandle.write(data, size, offset) except CacheRetry: e = FuseOSError(EAGAIN) e.strerror = "Timeout waiting for file write" vos.logger.debug("Timeout Waiting for file write: %s", path) raise e
def getNode(self, path, force=False, limit=0): """Use the client and pull the node from VOSpace. Currently force=False is the default... so we never check VOSpace to see if the node metadata is different from what we have. This doesn't keep the node metadata current but is faster if VOSpace is slow. """ vos.logger.debug("force? -> %s path -> %s" % (force, path)) ## Pull the node meta data from VOSpace. try: vos.logger.debug("requesting node %s from VOSpace" % (path)) node = self.client.getNode(path, force=force, limit=limit) except Exception as e: vos.logger.debug(str(e)) vos.logger.debug(type(e)) ex = FuseOSError(getattr(e, 'errno', ENOENT)) ex.filename = path ex.strerror = getattr(e, 'strerror', 'Error getting %s' % (path)) vos.logger.debug("failing with errno = %d" % ex.errno) raise ex return node
def chmod(self, path, mode): """ Set the read/write groups on the VOSpace node based on chmod style modes. This function is a bit funny as the VOSpace spec sets the name of the read and write groups instead of having mode setting as a separate action. A chmod that adds group permission thus becomes a chgrp action. Here I use the logic that the new group will be inherited from the container group information. """ vos.logger.debug("Changing mode for %s to %d" % (path, mode)) node = self.getNode(path) parent = self.getNode(os.path.dirname(path)) if node.groupread == "NONE": node.groupread = parent.groupread if node.groupwrite == "NONE": node.groupwrite = parent.groupwrite # The 'node' object returned by getNode has a chmod method # that we now call to set the mod, since we set the group(s) # above. NOTE: If the parrent doesn't have group then NONE is # passed up and the groupwrite and groupread will be set tovos: # the string NONE. if node.chmod(mode): # Now set the time of change/modification on the path... # TODO: This has to be broken. Attributes may come from Cache if # the file is modified. Even if they don't come from the cache, # the getAttr method calls getNode with force=True, which returns # a different Node object than "node". The st_ctime value will be # updated on the newly replaced node in self.node[path] but # not in node, then node is pushed to vospace without the st_time # change, and then it is pulled back, overwriting the change that # was made in self.node[path]. Plus modifying the mtime of the file # is not conventional Unix behaviour. The mtime of the parent # directory would be changed. self.getattr(path)['st_ctime'] = time.time() ## if node.chmod returns false then no changes were made. try: self.client.update(node) self.getNode(path, force=True) except Exception as e: vos.logger.debug(str(e)) vos.logger.debug(type(e)) e = FuseOSError(getattr(e, 'errno', EIO)) e.filename = path e.strerror = getattr(e, 'strerror', 'failed to chmod on %s' % (path)) raise e
def __init__(self, root, cache_dir, options, conn=None, cache_limit=1024, cache_nodes=False, cache_max_flush_threads=10, secure_get=False): """Initialize the VOFS. cache_limit is in MB. The style here is to use dictionaries to contain information about the Node. The full VOSpace path is used as the Key for most of these dictionaries.""" self.cache_nodes = cache_nodes # Standard attributes of the Node # Where in the file system this Node is currently located self.loading_dir = {} # Command line options. self.opt = options # What is the 'root' of the VOSpace? (eg vos:MyVOSpace) self.root = root # VOSpace is a bit slow so we do some caching. self.cache = Cache(cache_dir, cache_limit, False, VOFS.cacheTimeout, maxFlushThreads=cache_max_flush_threads) # All communication with the VOSpace goes through this client # connection. try: self.client = vos.Client(root_node=root, conn=conn, transfer_shortcut=True, secure_get=secure_get) except Exception as e: e = FuseOSError(getattr(e, 'errno', EIO)) e.filename = root e.strerror = getattr(e, 'strerror', 'failed while making mount') raise e # Create a condition variable to get rid of those nasty sleeps self.condition = CacheCondition(lock=None, timeout=VOFS.cacheTimeout)
def readdir(self, path, id): """Send a list of entries in this directory""" vos.logger.debug("Getting directory list for %s " % (path)) ## reading from VOSpace can be slow, we'll do this in a thread import thread with self.condition: if not self.loading_dir.get(path, False): self.loading_dir[path] = True thread.start_new_thread(self.load_dir, (path, )) while self.loading_dir.get(path, False): vos.logger.debug("Waiting ... ") try: self.condition.wait() except CacheRetry: e = FuseOSError(EAGAIN) e.strerror = "Timeout waiting for directory listing" vos.logger.debug("Timeout Waiting for directory read: %s", path) raise e return ['.', '..'] + [e.name.encode('utf-8') for e in self.getNode( path, force=False, limit=None).getNodeList()]
def unlink(self, path): fn = ''.join([self.root, path.lstrip('/')]) try: self.fs.rm(fn, False) except (IOError, FileNotFoundError): raise FuseOSError(EIO)
def open(self, path, flags, *mode): """Open file with the desired modes Here we return a handle to the cached version of the file which is updated if older and out of sync with VOSpace. """ logger.debug("Opening %s with flags %s" % (path, flag2mode(flags))) node = None # according to man for open(2), flags must contain one of O_RDWR, # O_WRONLY or O_RDONLY. Because O_RDONLY=0 and options other than # O_RDWR, O_WRONLY and O_RDONLY may be present, # readonly = (flags == O_RDONLY) and readonly = (flags | O_RDONLY) # won't work. The only way to detect if it's a read only is to check # whether the other two flags are absent. read_only = ((flags & (os.O_RDWR | os.O_WRONLY)) == 0) must_exist = not ((flags & os.O_CREAT) == os.O_CREAT) cache_file_attrs = self.cache.getAttr(path) if cache_file_attrs is None and not read_only: # file in the cache not in the process of being modified. # see if this node already exists in the cache; if not get info # from vospace try: node = self.getNode(path) except IOError as e: if e.errno == 404: # file does not exist if not flags & os.O_CREAT: # file doesn't exist unless created raise FuseOSError(ENOENT) else: raise FuseOSError(e.errno) # check if this file is locked, if locked on vospace then don't open locked = False if node and node.props.get('islocked', False): logger.debug("%s is locked." % path) locked = True if not read_only and node and not locked: if node.type == "vos:DataNode": parent_node = self.getNode(os.path.dirname(path), force=False, limit=1) if parent_node.props.get('islocked', False): logger.debug("%s is locked by parent node." % path) locked = True elif node.type == "vos:LinkNode": try: # sometimes targetNodes aren't internal... so then not # locked target_node = self.getNode(node.target, force=False, limit=1) if target_node.props.get('islocked', False): logger.debug("{0} target node is locked.".format(path)) locked = True else: target_parent_node = self.getNode(os.path.dirname( node.target), force=False, limit=1) if target_parent_node.props.get('islocked', False): logger.debug( "{0} parent node is locked.".format(path)) locked = True except Exception as lock_exception: logger.warn("Error while checking for lock: {0}".format( str(lock_exception))) pass if locked and not read_only: # file is locked, cannot write e = OSError(EPERM) e.filename = path e.strerror = "Cannot write to locked file" logger.debug("{0}".format(e)) raise e my_proxy = MyIOProxy(self, path) if node is not None: my_proxy.set_size(int(node.props.get('length'))) my_proxy.set_md5(node.props.get('MD5')) logger.debug("IO Proxy initialized:{0} in backing.".format(my_proxy)) # new file in cache library or if no node information (node not in vospace). handle = self.cache.open(path, flags & os.O_WRONLY != 0, must_exist, my_proxy, self.cache_nodes) logger.debug("Creating file:{0} in backing.".format(path)) if flags & os.O_TRUNC != 0: handle.truncate(0) if node is not None: handle.setHeader(my_proxy.getSize(), my_proxy.get_md5()) return HandleWrapper(handle, read_only).get_id()
def access(self, path, mode): if not os.access(path, mode): raise FuseOSError(EACCES)
def readlink(self, filepath): raise FuseOSError(EPERM)
def find_name_in_dir(self, cur_dir, name): for entry in cur_dir.extent_as_dir.entries: entry_body = getattr(entry, "body", None) if entry_body is not None and entry_body.file_name == name: return entry_body raise FuseOSError(errno.ENOENT)
def chmod(self, path, mode): if mode == self.note_stat['st_mode']: return 0 else: logging.error('chmod is disabled.') raise FuseOSError(errno.EPERM)
def getattr(self, path, fh=None): if path not in self.files: raise FuseOSError(errno.ENOENT) with configLock: return self.files[path]
def mknod ( self, path, mode, dev ): #self.log.info( 'CALL mknod {} {} {}'.format(path, oct(mode), dev)) raise FuseOSError(errno.EROFS)
def mkdir ( self, path, mode ): #self.log.info( 'CALL mkdir {} {}'.format(path, oct(mode))) raise FuseOSError(errno.EROFS)
def fsync ( self, path, isFsyncFile ): #self.log.info( 'CALL fsync {} {}'.format(path, isFsyncFile)) raise FuseOSError(errno.EROFS)
def chown ( self, path, uid, gid ): #self.log.info( 'CALL chown {} {} {}'.format(path, uid, gid)) raise FuseOSError(errno.EROFS)
def mythread ( self ): #self.log.info('mythread') raise FuseOSError(errno.ENOSYS)
def getattr(self, path, fh=None): """ - st_mode (protection bits) - st_ino (inode number) - st_dev (device) - st_nlink (number of hard links) - st_uid (user ID of owner) - st_gid (group ID of owner) - st_size (size of file, in bytes) - st_atime (time of most recent access) - st_mtime (time of most recent content modification) - st_ctime (platform dependent; time of most recent metadata change on Unix, or the time of creation on Windows). """ #self.log.info("CALL getattr: %s", path) path = self.fsEncodeName(path) depth = getDepth(path) # depth of path, zero-based from root if depth == 0: # Fake the root target = self.lastBackupSet(False) timestamp = float(target['starttime']) st = { 'st_mode': stat.S_IFDIR | 0o555, 'st_ino': 0, 'st_dev': 0, 'st_nlink': 32, 'st_uid': 0, 'st_gid': 0, 'st_size': 4096, 'st_atime': timestamp, 'st_mtime': timestamp, 'st_ctime': timestamp, } return st elif depth == 1: # Root directory contents lead = getParts(path) if lead[0] == self.current: target = self.lastBackupSet(True) timestamp = float(target['endtime']) st = { 'st_mode': stat.S_IFLNK | 0o755, 'st_ino': 1, 'st_dev': 0, 'st_nlink': 1, 'st_uid': 0, 'st_gid': 0, 'st_size': 4096, 'st_atime': timestamp, 'st_mtime': timestamp, 'st_ctime': timestamp } return st else: f = self.getBackupSetInfo(lead[0]) #self.log.debug("Got backupset info for %s: %s", lead[0], str(f)) if f: timestamp = float(f['starttime']) st = { 'st_mode': stat.S_IFDIR | 0o555, 'st_ino': int(float(f['starttime'])), 'st_dev': 0, 'st_nlink': 2, 'st_uid': 0, 'st_gid': 0, 'st_size': 4096, 'st_atime': timestamp, 'st_mtime': timestamp, 'st_ctime': timestamp } return st else: f = self.getFileInfoByPath(path) if f: st = { 'st_mode': f["mode"], 'st_ino': f["inode"], 'st_dev': 0, 'st_nlink': f["nlinks"], 'st_uid': f["uid"], 'st_gid': f["gid"], 'st_atime': f["mtime"], 'st_mtime': f["mtime"], 'st_ctime': f["ctime"] } if f["size"] is not None: st['st_size'] = int(f["size"]) elif f["dir"]: st['st_size'] = 4096 # Arbitrary number else: st['st_size'] = 0 return st logger.debug("File not found: %s", path) raise FuseOSError(errno.ENOENT)
def getattr(self, path, fh=None): path = path.encode('utf-8') if debug == True: appLog('debug', 'Called: getattr() - Path: ' + path) # Get userid and groupid for current user. uid = pwd.getpwuid(os.getuid()).pw_uid gid = pwd.getpwuid(os.getuid()).pw_gid # Get current time. now = int(time()) # Check wether data exists for item. item = self.getDropboxMetadata(path) if item == False: raise FuseOSError(ENOENT) # Handle last modified times. if 'client_modified' in item: modified = str(item['client_modified']) #2012-08-11T12:41:30Z if modified.endswith('Z'): modified = mktime( datetime.strptime(modified, '%Y-%m-%dT%H:%M:%SZ').timetuple()) else: modified = mktime( datetime.strptime(modified, '%Y-%m-%d %H:%M:%S').timetuple()) else: modified = int(now) if debug == True: appLog('debug', "item: " + str(item)) if 'entries' in item or item['.tag'] == 'folder': # Get st_nlink count for directory. properties = dict(st_mode=S_IFDIR | 0755, st_size=0, st_ctime=modified, st_mtime=modified, st_atime=now, st_uid=uid, st_gid=gid, st_nlink=2) if debug == True: appLog( 'debug', 'Returning properties for directory: ' + path + ' (' + str(properties) + ')') return properties elif item['.tag'] == 'file': properties = dict( st_mode=S_IFREG | 0644, st_size=item['size'], st_ctime=modified, st_mtime=modified, st_atime=now, st_uid=uid, st_gid=gid, st_nlink=1, ) if debug == True: appLog( 'debug', 'Returning properties for file: ' + path + ' (' + str(properties) + ')') return properties
def write(self, path, buf, offset, fh): path = path.encode('utf-8') if debug == True: appLog( 'debug', 'Called: write() - Path: ' + path + ' Offset: ' + str(offset) + ' FH: ' + str(fh)) try: # Check for the beginning of the file. if fh in self.openfh: if self.openfh[fh]['f'] == False: if debug == True: appLog('debug', 'Uploading first chunk to Dropbox...') # Check if the write request exceeds the maximum buffer size. if len(buf) >= write_cache or len(buf) < 4096: if debug == True: appLog( 'debug', 'Cache exceeds configured write_cache. Uploading...' ) result = self.dbxChunkedUpload(buf, "", 0) self.openfh[fh]['f'] = { 'upload_id': result['upload_id'], 'offset': result['offset'], 'buf': '' } else: if debug == True: appLog( 'debug', 'Buffer does not exceed configured write_cache. Caching...' ) self.openfh[fh]['f'] = { 'upload_id': '', 'offset': 0, 'buf': buf } return len(buf) else: if debug == True: appLog('debug', 'Uploading another chunk to Dropbox...') if len(buf) + len(self.openfh[fh]['f']['buf'] ) >= write_cache or len(buf) < 4096: if debug == True: appLog( 'debug', 'Cache exceeds configured write_cache. Uploading...' ) result = self.dbxChunkedUpload( self.openfh[fh]['f']['buf'] + buf, self.openfh[fh]['f']['upload_id'], self.openfh[fh]['f']['offset']) self.openfh[fh]['f'] = { 'upload_id': result['upload_id'], 'offset': result['offset'], 'buf': '' } else: if debug == True: appLog( 'debug', 'Buffer does not exceed configured write_cache. Caching...' ) self.openfh[fh]['f'].update( {'buf': self.openfh[fh]['f']['buf'] + buf}) return len(buf) else: raise FuseOSError(EIO) except Exception, e: appLog('error', 'Could not write to remote file: ' + path, traceback.format_exc()) raise FuseOSError(EIO)
def getattr(self, path, fh=None): node = self.tree.find_path(path) if node is None: raise FuseOSError(ENOENT) return node.get_attrs()
def rename ( self, oldPath, newPath ): #self.log.info('CALL rename {} {}'.format(oldPath, newPath)) raise FuseOSError(errno.EROFS)
def getattr(self, path, fh=None): if path not in self.files: raise FuseOSError(ENOENT) return self.files[path]
def rmdir ( self, path ): #self.log.info('CALL rmdir {}'.format(path)) raise FuseOSError(errno.EROFS)
def chown(self, path, uid, gid): logging.error('chown is disabled.') raise FuseOSError(errno.EPERM)
def symlink ( self, targetPath, linkPath ): #self.log.info('CALL symlink {} {}'.format(path, linkPath)) raise FuseOSError(errno.EROFS)
def symlink(self, target, source): raise FuseOSError(EPERM)
def truncate ( self, path, size ): #self.log.info('CALL truncate {} {}'.format(path, size)) raise FuseOSError(errno.EROFS)
def unlink(self, file_path): """Remove a file.""" # TODO: Change to simply move to "trash". Have a FUSE option to elect this # behavior. path_relations = PathRelations.get_instance() try: entry_clause = path_relations.get_clause_from_path(file_path) except GdNotFoundError: _logger.exception("Could not process [%s] (unlink).", file_path) raise FuseOSError(ENOENT) except: _logger.exception( "Could not get clause from file-path [%s] " "(unlink).", file_path) raise FuseOSError(EIO) if not entry_clause: _logger.error("Path [%s] does not exist for unlink().", file_path) raise FuseOSError(ENOENT) entry_id = entry_clause[CLAUSE_ID] normalized_entry = entry_clause[CLAUSE_ENTRY] # Check if a directory. if normalized_entry.is_directory: _logger.error( "Can not unlink() directory [%s] with ID [%s]. " "Must be file.", file_path, entry_id) raise FuseOSError(EISDIR) # Remove online. Complements local removal (if not found locally, a # follow-up request checks online). gd = get_gdrive() try: gd.remove_entry(normalized_entry) except NameError: raise FuseOSError(ENOENT) except: _logger.exception("Could not remove file [%s] with ID [%s].", file_path, entry_id) raise FuseOSError(EIO) # Remove from cache. Will no longer be able to be found, locally. PathRelations.get_instance().remove_entry_all(entry_id) # Remove from among opened-files. om = gdrivefs.opened_file.get_om() try: opened_file = om.remove_by_filepath(file_path) except: _logger.exception( "There was an error while removing all " "opened-file instances for file [%s] " "(remove).", file_path) raise FuseOSError(EIO)
def open(self, path, flags, *mode): """Open file with the desired modes Here we return a handle to the cached version of the file which is updated if older and out of synce with VOSpace. """ vos.logger.debug("Opening %s with flags %s" % (path, flag2mode(flags))) node = None # according to man for open(2), flags must contain one of O_RDWR, # O_WRONLY or O_RDONLY. Because O_RDONLY=0 and options other than # O_RDWR, O_WRONLY and O_RDONLY may be present, # readonly = (flags == O_RDONLY) and readonly = (flags | # O_RDONLY) # won't work. The only way to detect if it's a read only is to check # whether the other two flags are absent. readOnly = ((flags & (os.O_RDWR | os.O_WRONLY)) == 0) mustExist = not ((flags & os.O_CREAT) == os.O_CREAT) cacheFileAttrs = self.cache.getAttr(path) if cacheFileAttrs is None and not readOnly: # file in the cache not in the process of being modified. # see if this node already exists in the cache; if not get info # from vospace try: node = self.getNode(path) except IOError as e: if e.errno == 404: # file does not exist if not flags & os.O_CREAT: # file doesn't exist unless created raise FuseOSError(ENOENT) else: raise FuseOSError(e.errno) ### check if this file is locked, if locked on vospace then don't open locked = False if node and node.props.get('islocked', False): vos.logger.debug("%s is locked." % path) locked = True if not readOnly and node and not locked: if node.type == "vos:DataNode": parentNode = self.getNode(os.path.dirname(path), force=False, limit=1) if parentNode.props.get('islocked', False): vos.logger.debug("%s is locked by parent node." % path) locked = True elif node.type == "vos:LinkNode": try: # sometimes targetNodes aren't internal... so then not # locked targetNode = self.getNode(node.target, force=False, limit=1) if targetNode.props.get('islocked', False): vos.logger.debug("%s target node is locked." % path) locked = True else: targetParentNode = self.getNode(os.path.dirname( node.target), force=False, limit=1) if targetParentNode.props.get('islocked', False): vos.logger.debug( "%s is locked by target parent node." % path) locked = True except Exception as e: vos.logger.error("Got an error while checking for lock: " + str(e)) pass if locked and not readOnly: # file is locked, cannot write e = FuseOSError(ENOENT) e.strerror = "Cannot create locked file" vos.logger.debug("Cannot create locked file: %s", path) raise e myProxy = MyIOProxy(self, path) if node is not None: myProxy.setSize(int(node.props.get('length'))) myProxy.setMD5(node.props.get('MD5')) # new file in cache library or if no node information (node not in # vospace). handle = self.cache.open(path, flags & os.O_WRONLY != 0, mustExist, myProxy, self.cache_nodes) if flags & os.O_TRUNC != 0: handle.truncate(0) if node is not None: handle.setHeader(myProxy.getSize(), myProxy.getMD5()) return HandleWrapper(handle, readOnly).getId()
def access(self, path, mode): full_path = self._full_path(path) if not os.access(full_path, mode): raise FuseOSError(errno.EACCES)
def write ( self, path, buf, offset ): #self.log.info('CALL write {} {} {}'.format(path, offset, len(buf))) raise FuseOSError(errno.EROFS)
def rmdir(self, path): node = self.getNode(path) if node and node.props.get('islocked', False): logger.debug("%s is locked." % path) raise FuseOSError(EPERM) self.client.delete(path)
def read(self, path, size, offset, fh): node = self.tree.find_path(path) if node is None: raise FuseOSError(EIO) return node.content[offset:offset + size]
def unlink ( self, path ): #self.log.info('CALL unlink {}'.format(path)) raise FuseOSError(errno.EROFS)
def readdir(self, path, fh): node = self.tree.find_path(path) if node is None: raise FuseOSError(EROFS) return ['.', '..'] + [child for child in node.children]