def test_init(self): auth_db = SwiftBasedAuthDB('http://127.0.0.1:8080/v1/auth') self.assertEquals(auth_db.auth_url, 'http://127.0.0.1:8080/v1/auth') self.assertEquals(auth_db.global_max_concurrency, 100) self.assertEquals(auth_db.max_concurrency, 10) self.assertEquals(auth_db.timeout, 260) self.assertEquals(auth_db.verbose, False) self.assertEquals(auth_db.rewrite_scheme, None) self.assertEquals(auth_db.rewrite_netloc, None) auth_db = SwiftBasedAuthDB( 'http://127.0.0.1:8080/v1/auth', global_max_concurrency=200, max_concurrency=20, timeout=460, verbose=True, rewrite_scheme='https', rewrite_netloc='some-hostname:1234', ) self.assertEquals(auth_db.auth_url, 'http://127.0.0.1:8080/v1/auth') self.assertEquals(auth_db.global_max_concurrency, 200) self.assertEquals(auth_db.max_concurrency, 20) self.assertEquals(auth_db.timeout, 460) self.assertEquals(auth_db.verbose, True) self.assertEquals(auth_db.rewrite_scheme, 'https') self.assertEquals(auth_db.rewrite_netloc, 'some-hostname:1234')
def test_no_storage_url(self): swift_conn = MagicMock() swift_conn.storage_url = 'http://some-storage-url/v1/AUTH_12345' auth_db = SwiftBasedAuthDB('http://127.0.0.1:8080/v1/auth') auth_db._rewrite_storage_url(swift_conn) self.assertEquals(swift_conn.storage_url, 'http://some-storage-url/v1/AUTH_12345')
def test_hostname_port(self): swift_conn = MagicMock() swift_conn.storage_url = 'http://some-storage-url/v1/AUTH_12345' auth_db = SwiftBasedAuthDB('http://127.0.0.1:8080/v1/auth', rewrite_netloc='hostname:1234') auth_db._rewrite_storage_url(swift_conn) self.assertEquals(swift_conn.storage_url, 'http://hostname:1234/v1/AUTH_12345')
def test_hostname_port(self): swift_conn = MagicMock() swift_conn.storage_url = 'http://some-storage-url/v1/AUTH_12345' auth_db = SwiftBasedAuthDB( 'http://127.0.0.1:8080/v1/auth', rewrite_netloc='hostname:1234') auth_db._rewrite_storage_url(swift_conn) self.assertEquals(swift_conn.storage_url, 'http://hostname:1234/v1/AUTH_12345')
def test_zero_concurrency(self): auth_db = SwiftBasedAuthDB( 'http://127.0.0.1:8080/v1/auth', global_max_concurrency=0, max_concurrency=0, ) def check_connection(conn): self.assertEquals(conn.username, 'username') self.assertEquals(conn.api_key, 'password') # Default connection pool size per host is 2 self.assertEquals(conn.pool.maxPersistentPerHost, 2) self.assertEquals(conn.pool.persistent, False) self.assertEquals(conn.locks, []) creds = UsernamePassword('username', 'password') d = auth_db.requestAvatarId(creds) d.addCallback(check_connection) return d
class MetricCollectorTest(unittest.TestCase): def setUp(self): self.auth_db = SwiftBasedAuthDB('http://127.0.0.1:8080/v1/auth') def test_init(self): auth_db = SwiftBasedAuthDB('http://127.0.0.1:8080/v1/auth') self.assertEquals(auth_db.auth_url, 'http://127.0.0.1:8080/v1/auth') self.assertEquals(auth_db.global_max_concurrency, 100) self.assertEquals(auth_db.max_concurrency, 10) self.assertEquals(auth_db.timeout, 260) self.assertEquals(auth_db.verbose, False) auth_db = SwiftBasedAuthDB( 'http://127.0.0.1:8080/v1/auth', global_max_concurrency=200, max_concurrency=20, timeout=460, verbose=True, ) self.assertEquals(auth_db.auth_url, 'http://127.0.0.1:8080/v1/auth') self.assertEquals(auth_db.global_max_concurrency, 200) self.assertEquals(auth_db.max_concurrency, 20) self.assertEquals(auth_db.timeout, 460) self.assertEquals(auth_db.verbose, True) @patch('swftp.auth.ThrottledSwiftConnection.authenticate', authenticate_good) def test_request_avatar_id(self): creds = UsernamePassword('username', 'password') return self.auth_db.requestAvatarId(creds) @patch('swftp.auth.ThrottledSwiftConnection.authenticate', authenticate_bad) def test_request_avatar_id_fail(self): creds = UsernamePassword('username', 'password') d = self.auth_db.requestAvatarId(creds) return self.assertFailure(d, UnauthorizedLogin) def test_request_avatar_id_invalid_method(self): return self.assertFailure( self.auth_db.requestAvatarId('nope'), UnauthorizedLogin)
def setUp(self): self.auth_db = SwiftBasedAuthDB('http://127.0.0.1:8080/v1/auth')
class AuthenticateTest(unittest.TestCase): def setUp(self): self.auth_db = SwiftBasedAuthDB('http://127.0.0.1:8080/v1/auth') def test_init(self): auth_db = SwiftBasedAuthDB('http://127.0.0.1:8080/v1/auth') self.assertEquals(auth_db.auth_url, 'http://127.0.0.1:8080/v1/auth') self.assertEquals(auth_db.global_max_concurrency, 100) self.assertEquals(auth_db.max_concurrency, 10) self.assertEquals(auth_db.timeout, 260) self.assertEquals(auth_db.verbose, False) self.assertEquals(auth_db.rewrite_scheme, None) self.assertEquals(auth_db.rewrite_netloc, None) auth_db = SwiftBasedAuthDB( 'http://127.0.0.1:8080/v1/auth', global_max_concurrency=200, max_concurrency=20, timeout=460, verbose=True, rewrite_scheme='https', rewrite_netloc='some-hostname:1234', ) self.assertEquals(auth_db.auth_url, 'http://127.0.0.1:8080/v1/auth') self.assertEquals(auth_db.global_max_concurrency, 200) self.assertEquals(auth_db.max_concurrency, 20) self.assertEquals(auth_db.timeout, 460) self.assertEquals(auth_db.verbose, True) self.assertEquals(auth_db.rewrite_scheme, 'https') self.assertEquals(auth_db.rewrite_netloc, 'some-hostname:1234') @patch('swftp.auth.ThrottledSwiftConnection.authenticate', authenticate_good) def test_request_avatar_id(self): creds = UsernamePassword('username', 'password') return self.auth_db.requestAvatarId(creds) @patch('swftp.auth.ThrottledSwiftConnection.authenticate', authenticate_good) def test_zero_concurrency(self): auth_db = SwiftBasedAuthDB( 'http://127.0.0.1:8080/v1/auth', global_max_concurrency=0, max_concurrency=0, ) def check_connection(conn): self.assertEquals(conn.username, 'username') self.assertEquals(conn.api_key, 'password') # Default connection pool size per host is 2 self.assertEquals(conn.pool.maxPersistentPerHost, 2) self.assertEquals(conn.pool.persistent, False) self.assertEquals(conn.locks, []) creds = UsernamePassword('username', 'password') d = auth_db.requestAvatarId(creds) d.addCallback(check_connection) return d @patch('swftp.auth.ThrottledSwiftConnection.authenticate', authenticate_bad) def test_request_avatar_id_fail(self): creds = UsernamePassword('username', 'password') d = self.auth_db.requestAvatarId(creds) return self.assertFailure(d, UnauthorizedLogin) def test_request_avatar_id_invalid_method(self): return self.assertFailure( self.auth_db.requestAvatarId('nope'), UnauthorizedLogin)
def makeService(options): """ Makes a new swftp-sftp service. The only option is the config file location. See CONFIG_DEFAULTS for list of configuration options. """ from twisted.conch.ssh.connection import SSHConnection from twisted.conch.ssh.factory import SSHFactory from twisted.conch.ssh.keys import Key from twisted.cred.portal import Portal from swftp.realm import SwftpRealm from swftp.sftp.server import SwiftSSHUserAuthServer from swftp.auth import SwiftBasedAuthDB from swftp.utils import ( log_runtime_info, GLOBAL_METRICS, parse_key_value_config) c = get_config(options['config_file'], options) sftp_service = service.MultiService() # ensure timezone is GMT os.environ['TZ'] = 'GMT' time.tzset() print('Starting SwFTP-sftp %s' % VERSION) # Add statsd service if c.get('sftp', 'log_statsd_host'): try: from swftp.statsd import makeService as makeStatsdService makeStatsdService( c.get('sftp', 'log_statsd_host'), c.getint('sftp', 'log_statsd_port'), sample_rate=c.getfloat('sftp', 'log_statsd_sample_rate'), prefix=c.get('sftp', 'log_statsd_metric_prefix') ).setServiceParent(sftp_service) except ImportError: sys.stderr.write('Missing Statsd Module. Requires "txstatsd" \n') if c.get('sftp', 'stats_host'): from swftp.report import makeService as makeReportService known_fields = [ 'command.login', 'command.logout', 'command.gotVersion', 'command.openFile', 'command.removeFile', 'command.renameFile', 'command.makeDirectory', 'command.removeDirectory', 'command.openDirectory', 'command.getAttrs', ] + GLOBAL_METRICS makeReportService( c.get('sftp', 'stats_host'), c.getint('sftp', 'stats_port'), known_fields=known_fields ).setServiceParent(sftp_service) authdb = SwiftBasedAuthDB( c.get('sftp', 'auth_url'), global_max_concurrency=c.getint('sftp', 'num_persistent_connections'), max_concurrency=c.getint('sftp', 'num_connections_per_session'), timeout=c.getint('sftp', 'connection_timeout'), extra_headers=parse_key_value_config(c.get('sftp', 'extra_headers')), verbose=c.getboolean('sftp', 'verbose'), rewrite_scheme=c.get('sftp', 'rewrite_storage_scheme'), rewrite_netloc=c.get('sftp', 'rewrite_storage_netloc'), ) rabbitmq_hosts = c.get('rabbitmq', 'rabbitmq_hosts') rabbitmq_cluster = RabbitClusterClient([RabbitReplica(host, port) \ for host, port in [(h,int(p)) for h,p in [r.split(':') \ for r in rabbitmq_hosts.split(',')]]], \ c.get('rabbitmq', 'username'), \ c.get('rabbitmq', 'password')) \ if rabbitmq_hosts else None queue_name = c.get('rabbitmq', 'queue_name') realm = SwftpRealm(rabbitmq_cluster, queue_name) sftpportal = Portal(realm) sftpportal.registerChecker(authdb) sshfactory = SSHFactory() protocol = SwiftSSHServerTransport protocol.maxConnectionsPerUser = c.getint('sftp', 'sessions_per_user') protocol.supportedCiphers = c.get('sftp', 'chiphers') protocol.supportedMACs = c.get('sftp', 'macs') protocol.supportedCompressions = c.get('sftp', 'compressions') sshfactory.protocol = protocol sshfactory.noisy = False sshfactory.portal = sftpportal sshfactory.services['ssh-userauth'] = SwiftSSHUserAuthServer sshfactory.services['ssh-connection'] = SSHConnection pub_key_string = file(c.get('sftp', 'pub_key')).read() priv_key_string = file(c.get('sftp', 'priv_key')).read() sshfactory.publicKeys = { 'ssh-rsa': Key.fromString(data=pub_key_string)} sshfactory.privateKeys = { 'ssh-rsa': Key.fromString(data=priv_key_string)} signal.signal(signal.SIGUSR1, log_runtime_info) signal.signal(signal.SIGUSR2, log_runtime_info) internet.TCPServer( c.getint('sftp', 'port'), sshfactory, interface=c.get('sftp', 'host')).setServiceParent(sftp_service) return sftp_service
def makeService(options): """ Makes a new swftp-ftp service. The only option is the config file location. See CONFIG_DEFAULTS for list of configuration options. """ from twisted.protocols.ftp import FTPFactory from twisted.cred.portal import Portal from swftp.ftp.server import SwftpFTPProtocol from swftp.realm import SwftpRealm from swftp.auth import SwiftBasedAuthDB from swftp.utils import (log_runtime_info, GLOBAL_METRICS, parse_key_value_config) print('Starting SwFTP-ftp %s' % VERSION) c = get_config(options['config_file'], options) ftp_service = service.MultiService() # Add statsd service if c.get('ftp', 'log_statsd_host'): try: from swftp.statsd import makeService as makeStatsdService makeStatsdService( c.get('ftp', 'log_statsd_host'), c.getint('ftp', 'log_statsd_port'), sample_rate=c.getfloat('ftp', 'log_statsd_sample_rate'), prefix=c.get( 'ftp', 'log_statsd_metric_prefix')).setServiceParent(ftp_service) except ImportError: sys.stderr.write('Missing Statsd Module. Requires "txstatsd" \n') if c.get('ftp', 'stats_host'): from swftp.report import makeService as makeReportService known_fields = [ 'command.login', 'command.logout', 'command.makeDirectory', 'command.removeDirectory', 'command.removeFile', 'command.rename', 'command.access', 'command.stat', 'command.list', 'command.openForReading', 'command.openForWriting', ] + GLOBAL_METRICS makeReportService( c.get('ftp', 'stats_host'), c.getint('ftp', 'stats_port'), known_fields=known_fields).setServiceParent(ftp_service) authdb = SwiftBasedAuthDB( c.get('ftp', 'auth_url'), global_max_concurrency=c.getint('ftp', 'num_persistent_connections'), max_concurrency=c.getint('ftp', 'num_connections_per_session'), timeout=c.getint('ftp', 'connection_timeout'), extra_headers=parse_key_value_config(c.get('ftp', 'extra_headers')), verbose=c.getboolean('ftp', 'verbose'), rewrite_scheme=c.get('ftp', 'rewrite_storage_scheme'), rewrite_netloc=c.get('ftp', 'rewrite_storage_netloc'), ) ftpportal = Portal(SwftpRealm()) ftpportal.registerChecker(authdb) ftpfactory = FTPFactory(ftpportal) protocol = SwftpFTPProtocol protocol.maxConnectionsPerUser = c.getint('ftp', 'sessions_per_user') ftpfactory.protocol = protocol ftpfactory.welcomeMessage = c.get('ftp', 'welcome_message') ftpfactory.allowAnonymous = False ftpfactory.timeOut = c.getint('ftp', 'session_timeout') signal.signal(signal.SIGUSR1, log_runtime_info) signal.signal(signal.SIGUSR2, log_runtime_info) internet.TCPServer(c.getint('ftp', 'port'), ftpfactory, interface=c.get('ftp', 'host')).setServiceParent(ftp_service) return ftp_service
class AuthenticateTest(unittest.TestCase): def setUp(self): self.auth_db = SwiftBasedAuthDB('http://127.0.0.1:8080/v1/auth') def test_init(self): auth_db = SwiftBasedAuthDB('http://127.0.0.1:8080/v1/auth') self.assertEquals(auth_db.auth_url, 'http://127.0.0.1:8080/v1/auth') self.assertEquals(auth_db.global_max_concurrency, 100) self.assertEquals(auth_db.max_concurrency, 10) self.assertEquals(auth_db.timeout, 260) self.assertEquals(auth_db.verbose, False) self.assertEquals(auth_db.rewrite_scheme, None) self.assertEquals(auth_db.rewrite_netloc, None) auth_db = SwiftBasedAuthDB( 'http://127.0.0.1:8080/v1/auth', global_max_concurrency=200, max_concurrency=20, timeout=460, verbose=True, rewrite_scheme='https', rewrite_netloc='some-hostname:1234', ) self.assertEquals(auth_db.auth_url, 'http://127.0.0.1:8080/v1/auth') self.assertEquals(auth_db.global_max_concurrency, 200) self.assertEquals(auth_db.max_concurrency, 20) self.assertEquals(auth_db.timeout, 460) self.assertEquals(auth_db.verbose, True) self.assertEquals(auth_db.rewrite_scheme, 'https') self.assertEquals(auth_db.rewrite_netloc, 'some-hostname:1234') @patch('swftp.auth.ThrottledSwiftConnection.authenticate', authenticate_good) def test_request_avatar_id(self): creds = UsernamePassword('username', 'password') return self.auth_db.requestAvatarId(creds) @patch('swftp.auth.ThrottledSwiftConnection.authenticate', authenticate_good) def test_zero_concurrency(self): auth_db = SwiftBasedAuthDB( 'http://127.0.0.1:8080/v1/auth', global_max_concurrency=0, max_concurrency=0, ) def check_connection(conn): self.assertEquals(conn.username, 'username') self.assertEquals(conn.api_key, 'password') # Default connection pool size per host is 2 self.assertEquals(conn.pool.maxPersistentPerHost, 2) self.assertEquals(conn.pool.persistent, False) self.assertEquals(conn.locks, []) creds = UsernamePassword('username', 'password') d = auth_db.requestAvatarId(creds) d.addCallback(check_connection) return d @patch('swftp.auth.ThrottledSwiftConnection.authenticate', authenticate_bad) def test_request_avatar_id_fail(self): creds = UsernamePassword('username', 'password') d = self.auth_db.requestAvatarId(creds) return self.assertFailure(d, UnauthorizedLogin) def test_request_avatar_id_invalid_method(self): return self.assertFailure(self.auth_db.requestAvatarId('nope'), UnauthorizedLogin)
def makeService(options): """ Makes a new swftp-sftp service. The only option is the config file location. See CONFIG_DEFAULTS for list of configuration options. """ from twisted.cred.portal import Portal from swftp.realm import SwftpRealm from swftp.auth import SwiftBasedAuthDB from swftp.utils import (log_runtime_info, GLOBAL_METRICS, parse_key_value_config) c = get_config(options['config_file'], options) smtp_service = service.MultiService() # ensure timezone is GMT os.environ['TZ'] = 'GMT' time.tzset() print('Starting SwFTP-smtp %s' % VERSION) authdb = SwiftBasedAuthDB( c.get('smtp', 'auth_url'), global_max_concurrency=c.getint('smtp', 'num_persistent_connections'), max_concurrency=c.getint('smtp', 'num_connections_per_session'), timeout=c.getint('smtp', 'connection_timeout'), extra_headers=parse_key_value_config(c.get('smtp', 'extra_headers')), verbose=c.getboolean('smtp', 'verbose'), rewrite_scheme=c.get('smtp', 'rewrite_storage_scheme'), rewrite_netloc=c.get('smtp', 'rewrite_storage_netloc'), ) rabbitmq_hosts = c.get('rabbitmq', 'rabbitmq_hosts') rabbitmq_cluster = RabbitClusterClient([RabbitReplica(host, port) \ for host, port in [(h,int(p)) for h,p in [r.split(':') \ for r in rabbitmq_hosts.split(',')]]], \ c.get('rabbitmq', 'username'), \ c.get('rabbitmq', 'password')) \ if rabbitmq_hosts else None queue_name = c.get('smtp', 'queue_name') swift_username = c.get('smtp', 'swift_username') swift_password = c.get('smtp', 'swift_password') def swift_connect(): d = authdb.requestAvatarId( UsernamePassword(swift_username, swift_password)) def cb(connection): return connection def errback(failure): log.err("Swift connection failed: %s" % failure.type) raise SMTPServerError(451, "Internal server error") d.addCallbacks(cb, errback) return d @defer.inlineCallbacks def prepare_path(): swift_conn = yield swift_connect() swift_filesystem = SwiftFileSystem(swift_conn) try: yield swift_filesystem.get_container_listing('smtp', '/') except swift.NotFound: yield swift_filesystem.makeDirectory('/smtp/') defer.returnValue(None) prepare_path() smtp_factory = SwftpSMTPFactory(swift_connect, rabbitmq_cluster,\ queue_name) signal.signal(signal.SIGUSR1, log_runtime_info) signal.signal(signal.SIGUSR2, log_runtime_info) internet.TCPServer(c.getint('smtp', 'port'), smtp_factory, interface=c.get('smtp', 'host')).setServiceParent(smtp_service) return smtp_service