def validateTo(self, user): recipient = str(user.dest) valid = yield self.isRecipientValid(recipient) if not valid: raise SMTPBadRcpt(user) swift_filesystem = SwiftFileSystem(self.swift_connection) path = '/smtp/%s/%s' % (recipient, uuid4()) d, swift_file = swift_filesystem.startFileUpload(path) message = SwiftMessage(d, swift_file) @defer.inlineCallbacks def onUpload(ignored): if not self.rabbitmq_cluster or not self.queue_name: defer.returnValue(None) replica = yield self.rabbitmq_cluster.connect() yield replica.send( self.queue_name, json.dumps({ 'username': self.swift_connection.username, 'path': path, 'origin': self.origin, 'recipient': recipient, 'subj': message.subject, 'gate': 'smtp' })) d.addCallback(onUpload) yield swift_file.started msg("Uploading %s" % path) defer.returnValue(lambda: message)
def validateTo(self, user): recipient = str(user.dest) valid = yield self.isRecipientValid(recipient) if not valid: raise SMTPBadRcpt(user) swift_filesystem = SwiftFileSystem(self.swift_connection) path = '/smtp/%s/%s' % (recipient, uuid4()) d, swift_file = swift_filesystem.startFileUpload(path) message = SwiftMessage(d, swift_file) @defer.inlineCallbacks def onUpload(ignored): if not self.rabbitmq_cluster or not self.queue_name: defer.returnValue(None) replica = yield self.rabbitmq_cluster.connect() yield replica.send(self.queue_name, json.dumps({ 'username': self.swift_connection.username, 'path': path, 'origin': self.origin, 'recipient': recipient, 'subj': message.subject, 'gate': 'smtp'})) d.addCallback(onUpload) yield swift_file.started msg("Uploading %s" % path) defer.returnValue(lambda: message)
def __init__(self, avatar): self.swiftconn = avatar.swiftconn self.swiftfilesystem = SwiftFileSystem(self.swiftconn) self.avatar = avatar self.rabbitmq_cluster = avatar.rabbitmq_cluster self.queue_name = avatar.queue_name self.conn = avatar.conn self.log_command('login')
def prepare_path(): swift_conn = yield swift_connect() swift_filesystem = SwiftFileSystem(swift_conn) try: yield swift_filesystem.get_container_listing('smtp', '/') except swift.NotFound: yield swift_filesystem.makeDirectory('/smtp/') defer.returnValue(None)
def isRecipientValid(self, recipient): valid = self.recipients.get(recipient) if valid is not None: msg("Recipient %s is %svalid [cached]" % (recipient, "" if valid else "not ")) defer.returnValue(valid) if not self.swift_connection: self.swift_connection = yield self.connectToSwift() swift_filesystem = SwiftFileSystem(self.swift_connection) try: yield swift_filesystem.getAttrs(''.join(['/smtp/',recipient])) valid = True except swift.NotFound: valid = False msg("Recipient %s is %svalid" % (recipient, "" if valid else "not ")) self.recipients[recipient] = valid defer.returnValue(valid)
def isRecipientValid(self, recipient): valid = self.recipients.get(recipient) if valid is not None: msg("Recipient %s is %svalid [cached]" % (recipient, "" if valid else "not ")) defer.returnValue(valid) if not self.swift_connection: self.swift_connection = yield self.connectToSwift() swift_filesystem = SwiftFileSystem(self.swift_connection) try: yield swift_filesystem.getAttrs(''.join(['/smtp/', recipient])) valid = True except swift.NotFound: valid = False msg("Recipient %s is %svalid" % (recipient, "" if valid else "not ")) self.recipients[recipient] = valid defer.returnValue(valid)
def remove_test_data(swift, prefix): swift_fs = SwiftFileSystem(swift) time.sleep(2) containers = yield swift_fs.get_account_listing() sem = defer.DeferredSemaphore(200) for container in containers: if container.startswith(prefix): while True: objs = yield list_all_objects(swift, container) dl = [] for obj in objs: dl.append(sem.run( swift.delete_object, container, obj['name'])) # Wait till all objects are done deleting yield DeferredList(dl, fireOnOneErrback=True) try: # Delete the container yield swift.delete_container(container) break except Conflict: # Retry listing if there are still objects # (this can happen a lot since the container server isn't # guarenteed to be consistent) pass
def __init__(self, swiftconn): self.swiftconn = swiftconn self.swiftfilesystem = SwiftFileSystem(self.swiftconn) self.log_command('login')
class SFTPServerForSwiftConchUser: "******" interface.implements(ISFTPServer) def __init__(self, avatar): self.swiftconn = avatar.swiftconn self.swiftfilesystem = SwiftFileSystem(self.swiftconn) self.avatar = avatar self.conn = avatar.conn self.log_command('login') def log_command(self, *args, **kwargs): return self.avatar.log_command(*args, **kwargs) def gotVersion(self, otherVersion, extData): return {} def openFile(self, fullpath, flags, attrs): self.log_command('openFile', fullpath, flags, attrs) f = SwiftFile(self, fullpath, flags=flags, attrs=attrs) d = f.checkExistance() def errback(failure): failure.trap(NotFound) raise SFTPError(FX_FAILURE, "Container Doesn't Exist") d.addCallback(lambda r: f) d.addErrback(errback) return d def removeFile(self, fullpath): self.log_command('removeFile', fullpath) return self.swiftfilesystem.removeFile(fullpath) def renameFile(self, oldpath, newpath): self.log_command('renameFile', oldpath, newpath) d = self.swiftfilesystem.renameFile(oldpath, newpath) def errback(failure): failure.trap(NotFound, Conflict) if failure.check(NotFound): raise SFTPError(FX_NO_SUCH_FILE, 'No Such File') if failure.check(Conflict): raise NotImplementedError d.addErrback(errback) return d def makeDirectory(self, fullpath, attrs): self.log_command('makeDirectory', fullpath, attrs) def errback(failure): failure.trap(NotFound) raise SFTPError(FX_NO_SUCH_FILE, 'Directory Not Found') d = self.swiftfilesystem.makeDirectory(fullpath, attrs) d.addErrback(errback) return d def removeDirectory(self, fullpath): self.log_command('removeDirectory', fullpath) d = self.swiftfilesystem.removeDirectory(fullpath) def errback(failure): failure.trap(NotFound, Conflict) if failure.check(NotFound): return if failure.check(Conflict): raise SFTPError(FX_FAILURE, 'Directory Not Empty') d.addErrback(errback) return d def openDirectory(self, fullpath): self.log_command('openDirectory', fullpath) directory = SwiftDirectory(self.swiftfilesystem, fullpath) def cb(*result): return directory def errback(failure): failure.trap(NotFound) raise SFTPError(FX_FAILURE, 'Not Found') d = directory.get_full_listing() d.addCallback(cb) d.addErrback(errback) return d def getAttrs(self, fullpath, followLinks=False): self.log_command('getAttrs', fullpath) d = self.swiftfilesystem.getAttrs(fullpath) def cb(result): return self.format_attrs(result) def errback(failure): failure.trap(NotFound) raise SFTPError(FX_NO_SUCH_FILE, 'Not Found') d.addCallback(cb) d.addErrback(errback) return d def format_attrs(self, result): s = swift_stat(**result) return { "size": s.st_size, "uid": s.st_uid, "gid": s.st_gid, "permissions": s.st_mode, "atime": int(s.st_atime), "mtime": int(s.st_mtime) } def setAttrs(self, path, attrs): return def readLink(self, path): raise NotImplementedError def makeLink(self, linkPath, targetPath): raise NotImplementedError def realPath(self, path): container, obj = obj_to_path(path) real_path = '/' if container: real_path += container if obj: real_path += '/' + obj return real_path def extendedRequest(self, extName, extData): raise NotImplementedError
def __init__(self, swiftconn, rabbitmq_cluster, queue_name): self.swiftconn = swiftconn self.swiftfilesystem = SwiftFileSystem(self.swiftconn) self.log_command('login') self.rabbitmq_cluster = rabbitmq_cluster self.queue_name = queue_name
class SFTPServerForSwiftConchUser(object): """ SFTP Server For a Swift User. Provides t.c.i.ISFTPServer :param avatar: SwiftSFTPUser instance """ interface.implements(ISFTPServer) def __init__(self, avatar): self.swiftconn = avatar.swiftconn self.swiftfilesystem = SwiftFileSystem(self.swiftconn) self.avatar = avatar self.rabbitmq_cluster = avatar.rabbitmq_cluster self.queue_name = avatar.queue_name self.conn = avatar.conn self.log_command('login') def log_command(self, *args, **kwargs): """ Logs the given command. :param \*args: args to log :param \*\*kwargs: kwargs to log """ return self.avatar.log_command(*args, **kwargs) def gotVersion(self, otherVersion, extData): """ Client sent their version info """ self.log_command('gotVersion', otherVersion, extData) return {} def openFile(self, fullpath, flags, attrs): """ Open File/Object. Checks for Object Existence :param str fullpath: path to an object :param flags: flags to open the object with :param dict attrs: extra attributes to open the object with """ self.log_command('openFile', fullpath, flags, attrs) f = SwiftFile(self, fullpath, flags=flags, attrs=attrs) d = f.checkExistance() def errback(failure): failure.trap(NotFound) raise SFTPError(FX_FAILURE, "Container Doesn't Exist") d.addCallback(lambda r: f) d.addErrback(errback) return d def removeFile(self, fullpath): """ Remove Object :param str fullpath: path to an object """ self.log_command('removeFile', fullpath) def errback(failure): failure.trap(NotFound) if failure.check(NotFound): return d = self.swiftfilesystem.removeFile(fullpath) d.addErrback(errback) return d def renameFile(self, oldpath, newpath): """ Rename an Object :param str oldpath: old path to an object :param str newpath: new path to an object """ self.log_command('renameFile', oldpath, newpath) d = self.swiftfilesystem.renameFile(oldpath, newpath) def errback(failure): failure.trap(NotFound, Conflict) if failure.check(NotFound): raise SFTPError(FX_NO_SUCH_FILE, 'No Such File') if failure.check(Conflict): raise NotImplementedError d.addErrback(errback) return d def makeDirectory(self, fullpath, attrs): """ Make a 'directory' (either container or object). The container must exist to create a directory object inside of it. :param str fullpath: path to the directory :param dict attrs: attributes to create the directory with """ self.log_command('makeDirectory', fullpath, attrs) def errback(failure): failure.trap(NotFound) raise SFTPError(FX_NO_SUCH_FILE, 'Directory Not Found') d = self.swiftfilesystem.makeDirectory(fullpath, attrs) d.addErrback(errback) return d def removeDirectory(self, fullpath): """ Remove a 'directory' (either container or object). Not recursive. Will not delete a non-empty container :param str fullpath: path to the directory """ self.log_command('removeDirectory', fullpath) d = self.swiftfilesystem.removeDirectory(fullpath) def errback(failure): failure.trap(NotFound, Conflict) if failure.check(NotFound): return if failure.check(Conflict): raise SFTPError(FX_FAILURE, 'Directory Not Empty') d.addErrback(errback) return d def openDirectory(self, fullpath): """ Open a 'directory' :param str fullpath: path to the directory """ self.log_command('openDirectory', fullpath) directory = SwiftDirectory(self.swiftfilesystem, fullpath) def cb(*result): return directory def errback(failure): failure.trap(NotFound) raise SFTPError(FX_FAILURE, 'Not Found') d = directory.get_full_listing() d.addCallback(cb) d.addErrback(errback) return d def getAttrs(self, fullpath, followLinks=False): """ Get attributes for an Object/Container :param str fullpath: path to the directory :param bool followLinks: whether or not to follow links (not used) """ self.log_command('getAttrs', fullpath) d = self.swiftfilesystem.getAttrs(fullpath) def cb(result): return self.format_attrs(result) def errback(failure): failure.trap(NotFound) raise SFTPError(FX_NO_SUCH_FILE, 'Not Found') d.addCallback(cb) d.addErrback(errback) return d def format_attrs(self, result): """ Helper for formatting getAttr results """ s = swift_stat(**result) return { "size": s.st_size, "uid": s.st_uid, "gid": s.st_gid, "permissions": s.st_mode, "atime": int(s.st_atime), "mtime": int(s.st_mtime) } def setAttrs(self, path, attrs): """ Set attributes on a container/object. No-Op """ return def readLink(self, path): """ No-Op """ raise NotImplementedError def makeLink(self, linkPath, targetPath): """ No-Op """ raise NotImplementedError def realPath(self, path): """ Normalizes a filepath """ container, obj = obj_to_path(path) real_path = '/' if container: real_path += container if obj: real_path += '/' + obj return real_path def extendedRequest(self, extName, extData): """ No-op """ raise NotImplementedError
class SwiftFTPShell(object): """ Implements all the methods needed to treat Swift as an FTP Shell """ implements(IFTPShell) def __init__(self, swiftconn, rabbitmq_cluster, queue_name): self.swiftconn = swiftconn self.swiftfilesystem = SwiftFileSystem(self.swiftconn) self.log_command('login') self.rabbitmq_cluster = rabbitmq_cluster self.queue_name = queue_name def log_command(self, command, *args): arg_list = ', '.join(str(arg) for arg in args) msg("cmd: %s(%s)" % (command, arg_list), system="SwFTP-FTP, (%s)" % self.swiftconn.username, metric='command.%s' % command) def username(self): return self.swiftconn.username def logout(self): self.log_command('logout') if self.swiftconn.pool: self.swiftconn.pool.closeCachedConnections() del self.swiftconn def _fullpath(self, path_parts): return '/'.join(path_parts) def makeDirectory(self, path): self.log_command('makeDirectory', path) fullpath = self._fullpath(path) return self.swiftfilesystem.makeDirectory(fullpath) def removeDirectory(self, path): self.log_command('removeDirectory', path) fullpath = self._fullpath(path) def not_found_eb(failure): failure.trap(NotFound) def conflict_eb(failure): failure.trap(Conflict) raise CmdNotImplementedForArgError( 'Cannot delete non-empty directories.') d = self.swiftfilesystem.removeDirectory(fullpath) d.addErrback(not_found_eb) d.addErrback(conflict_eb) return d def removeFile(self, path): self.log_command('removeFile', path) fullpath = self._fullpath(path) def errback(failure): failure.trap(NotFound, NotImplementedError) if failure.check(NotImplementedError): return defer.fail(IsADirectoryError(fullpath)) d = defer.maybeDeferred(self.swiftfilesystem.removeFile, fullpath) d.addErrback(errback) return d def rename(self, fromPath, toPath): self.log_command('rename', fromPath, toPath) oldpath = self._fullpath(fromPath) newpath = self._fullpath(toPath) d = self.swiftfilesystem.renameFile(oldpath, newpath) def errback(failure): failure.trap(NotFound, Conflict, NotImplementedError) if failure.check(NotFound): return defer.fail(FileNotFoundError(oldpath)) else: return defer.fail(CmdNotImplementedForArgError(oldpath)) d.addErrback(errback) return d def access(self, path): self.log_command('access', path) fullpath = self._fullpath(path) d = self.swiftfilesystem.getAttrs(fullpath) def cb(result): if result['content_type'] == 'application/directory': return defer.succeed(lambda: None) return defer.fail(IsNotADirectoryError(fullpath)) d.addCallback(cb) def err(failure): failure.trap(NotFound) # Containers need to actually exist before uploading anything # inside of them. Therefore require containers to actually exist. # All other paths don't have to. if len(path) != 1: return defer.succeed(lambda: None) else: return defer.fail(IsNotADirectoryError(fullpath)) d.addErrback(err) return d def stat(self, path, keys=()): self.log_command('stat', path, keys) fullpath = self._fullpath(path) def cb(result): return stat_format(keys, result) def err(failure): failure.trap(NotFound) return defer.fail(FileNotFoundError(fullpath)) d = self.swiftfilesystem.getAttrs(fullpath) d.addCallback(cb) d.addErrback(err) return d def list(self, path=None, keys=()): self.log_command('list', path) fullpath = self._fullpath(path) def cb(results): l = [] for key, value in results.iteritems(): l.append([key, stat_format(keys, value)]) return l def err(failure): failure.trap(NotFound) return defer.fail(FileNotFoundError(fullpath)) d = self.swiftfilesystem.get_full_listing(fullpath) d.addCallback(cb) d.addErrback(err) return d def openForReading(self, path): self.log_command('openForReading', path) fullpath = self._fullpath(path) def cb(results): return SwiftReadFile(self.swiftfilesystem, fullpath) def err(failure): failure.trap(NotFound) return defer.fail(FileNotFoundError(fullpath)) try: d = self.swiftfilesystem.checkFileExistance(fullpath) d.addCallback(cb) d.addErrback(err) return d except NotImplementedError: return defer.fail(IsADirectoryError(fullpath)) def openForWriting(self, path): self.log_command('openForWriting', path) fullpath = self._fullpath(path) container, obj = obj_to_path(fullpath) if not container or not obj: raise CmdNotImplementedForArgError( 'Cannot upload files to root directory.') f = SwiftWriteFile(self.swiftfilesystem, fullpath,\ self.rabbitmq_cluster, self.queue_name) return defer.succeed(f)
def __init__(self, avatar): self.swiftconn = avatar.swiftconn self.swiftfilesystem = SwiftFileSystem(self.swiftconn) self.avatar = avatar self.conn = avatar.conn self.log_command('login')
class SwiftFTPShell: """ Implements all the methods needed to treat Swift as an FTP Shell """ implements(IFTPShell) def __init__(self, swiftconn): self.swiftconn = swiftconn self.swiftfilesystem = SwiftFileSystem(self.swiftconn) self.log_command('login') def log_command(self, command, *args): arg_list = ', '.join(str(arg) for arg in args) log.msg("COMMAND: %s(%s)" % (command, arg_list), system="SwFTP-FTP, (%s)" % self.swiftconn.username, metric='command.%s' % command) def logout(self): self.log_command('logout') def _fullpath(self, path_parts): return '/'.join(path_parts) def makeDirectory(self, path): self.log_command('makeDirectory', path) fullpath = self._fullpath(path) return self.swiftfilesystem.makeDirectory(fullpath) def removeDirectory(self, path): self.log_command('removeDirectory', path) fullpath = self._fullpath(path) def not_found_eb(failure): failure.trap(NotFound) def conflict_eb(failure): failure.trap(Conflict) raise CmdNotImplementedForArgError( 'Cannot delete non-empty directories.') d = self.swiftfilesystem.removeDirectory(fullpath) d.addErrback(not_found_eb) d.addErrback(conflict_eb) return d def removeFile(self, path): self.log_command('removeFile', path) fullpath = self._fullpath(path) def errback(failure): failure.trap(NotFound) d = self.swiftfilesystem.removeFile(fullpath) d.addErrback(errback) return d def rename(self, fromPath, toPath): self.log_command('rename', fromPath, toPath) oldpath = self._fullpath(fromPath) newpath = self._fullpath(toPath) d = self.swiftfilesystem.renameFile(oldpath, newpath) def errback(failure): failure.trap(NotFound, Conflict, NotImplementedError) if failure.check(NotFound): return defer.fail(FileNotFoundError(oldpath)) else: return defer.fail(CmdNotImplementedForArgError(oldpath)) d.addErrback(errback) return d def access(self, path): self.log_command('access', path) fullpath = self._fullpath(path) d = self.swiftfilesystem.getAttrs(fullpath) def cb(result): if result['content_type'] == 'application/directory': return defer.succeed(lambda: None) return defer.fail(IsNotADirectoryError(fullpath)) d.addCallback(cb) def err(failure): failure.trap(NotFound) # Containers need to actually exist before uploading anything # inside of them. Therefore require containers to actually exist. # All other paths don't have to. if len(path) != 1: return defer.succeed(lambda: None) else: return defer.fail(IsNotADirectoryError(fullpath)) d.addErrback(err) return d def stat(self, path, keys=()): self.log_command('stat', path, keys) fullpath = self._fullpath(path) def cb(result): return stat_format(keys, result) def err(failure): failure.trap(NotFound) return defer.fail(FileNotFoundError(fullpath)) d = self.swiftfilesystem.getAttrs(fullpath) d.addCallback(cb) d.addErrback(err) return d def list(self, path=None, keys=()): self.log_command('list', path) fullpath = self._fullpath(path) def cb(results): l = [] for key, value in results.iteritems(): l.append([key, stat_format(keys, value)]) return l def err(failure): failure.trap(NotFound) return defer.fail(FileNotFoundError(fullpath)) d = self.swiftfilesystem.get_full_listing(fullpath) d.addCallback(cb) d.addErrback(err) return d def openForReading(self, path): self.log_command('openForReading', path) fullpath = self._fullpath(path) def cb(results): return SwiftReadFile(self.swiftfilesystem, fullpath) def err(failure): failure.trap(NotFound) return defer.fail(FileNotFoundError(fullpath)) try: d = self.swiftfilesystem.checkFileExistance(fullpath) d.addCallback(cb) d.addErrback(err) return d except NotImplementedError: return defer.fail(IsADirectoryError(fullpath)) def openForWriting(self, path): self.log_command('openForWriting', path) fullpath = self._fullpath(path) container, obj = obj_to_path(fullpath) if not container or not obj: log.msg('cannot upload to root') raise CmdNotImplementedForArgError( 'Cannot upload files to root directory.') f = SwiftWriteFile(self.swiftfilesystem, fullpath) return defer.succeed(f)
def __init__(self, swiftconn): self.swiftconn = swiftconn self.swiftfilesystem = SwiftFileSystem(self.swiftconn) self.log_command('login') log.msg(metric='num_clients')