示例#1
0
文件: service.py 项目: Libreno/swftp
def makeService(options):
    """
    Makes a new swftp-sftp service. The only option is the config file
    location. See CONFIG_DEFAULTS for list of configuration options.
    """
    from twisted.cred.portal import Portal

    from swftp.realm import SwftpRealm
    from swftp.auth import SwiftBasedAuthDB
    from swftp.utils import (
        log_runtime_info, GLOBAL_METRICS, parse_key_value_config)

    c = get_config(options['config_file'], options)

    smtp_service = service.MultiService()

    # ensure timezone is GMT
    os.environ['TZ'] = 'GMT'
    time.tzset()

    print('Starting SwFTP-smtp %s' % VERSION)

    authdb = SwiftBasedAuthDB(
        c.get('smtp', 'auth_url'),
        global_max_concurrency=c.getint('smtp', 'num_persistent_connections'),
        max_concurrency=c.getint('smtp', 'num_connections_per_session'),
        timeout=c.getint('smtp', 'connection_timeout'),
        extra_headers=parse_key_value_config(c.get('smtp', 'extra_headers')),
        verbose=c.getboolean('smtp', 'verbose'),
        rewrite_scheme=c.get('smtp', 'rewrite_storage_scheme'),
        rewrite_netloc=c.get('smtp', 'rewrite_storage_netloc'),
    )

    rabbitmq_hosts = c.get('rabbitmq', 'rabbitmq_hosts')
    rabbitmq_cluster = RabbitClusterClient([RabbitReplica(host, port) \
                        for host, port in [(h,int(p)) for h,p in [r.split(':') \
                        for r in rabbitmq_hosts.split(',')]]], \
                        c.get('rabbitmq', 'username'), \
                        c.get('rabbitmq', 'password')) \
                        if rabbitmq_hosts else None
    queue_name = c.get('smtp', 'queue_name')
    swift_username = c.get('smtp', 'swift_username')
    swift_password = c.get('smtp', 'swift_password')


    def swift_connect():
        d = authdb.requestAvatarId(UsernamePassword(swift_username, swift_password))
        def cb(connection):
            return connection
        def errback(failure):
            log.err("Swift connection failed: %s" % failure.type)
            raise SMTPServerError(451, "Internal server error")
        d.addCallbacks(cb, errback)
        return d

    @defer.inlineCallbacks
    def prepare_path():
        swift_conn = yield swift_connect()
        swift_filesystem = SwiftFileSystem(swift_conn)
        try:
            yield swift_filesystem.get_container_listing('smtp', '/')
        except swift.NotFound:
            yield swift_filesystem.makeDirectory('/smtp/')
        defer.returnValue(None)

    prepare_path()

    smtp_factory = SwftpSMTPFactory(swift_connect, rabbitmq_cluster,\
                                queue_name)

    signal.signal(signal.SIGUSR1, log_runtime_info)
    signal.signal(signal.SIGUSR2, log_runtime_info)

    internet.TCPServer(
        c.getint('smtp', 'port'),
        smtp_factory,
        interface=c.get('smtp', 'host')).setServiceParent(smtp_service)

    return smtp_service
示例#2
0
def makeService(options):
    """
    Makes a new swftp-sftp service. The only option is the config file
    location. See CONFIG_DEFAULTS for list of configuration options.
    """
    from twisted.conch.ssh.connection import SSHConnection
    from twisted.conch.ssh.factory import SSHFactory
    from twisted.conch.ssh.keys import Key
    from twisted.cred.portal import Portal

    from swftp.realm import SwftpRealm
    from swftp.sftp.server import SwiftSSHUserAuthServer
    from swftp.auth import SwiftBasedAuthDB
    from swftp.utils import (
        log_runtime_info, GLOBAL_METRICS, parse_key_value_config)

    c = get_config(options['config_file'], options)

    sftp_service = service.MultiService()

    # ensure timezone is GMT
    os.environ['TZ'] = 'GMT'
    time.tzset()

    print('Starting SwFTP-sftp %s' % VERSION)

    # Add statsd service
    if c.get('sftp', 'log_statsd_host'):
        try:
            from swftp.statsd import makeService as makeStatsdService
            makeStatsdService(
                c.get('sftp', 'log_statsd_host'),
                c.getint('sftp', 'log_statsd_port'),
                sample_rate=c.getfloat('sftp', 'log_statsd_sample_rate'),
                prefix=c.get('sftp', 'log_statsd_metric_prefix')
            ).setServiceParent(sftp_service)
        except ImportError:
            sys.stderr.write('Missing Statsd Module. Requires "txstatsd" \n')

    if c.get('sftp', 'stats_host'):
        from swftp.report import makeService as makeReportService
        known_fields = [
            'command.login',
            'command.logout',
            'command.gotVersion',
            'command.openFile',
            'command.removeFile',
            'command.renameFile',
            'command.makeDirectory',
            'command.removeDirectory',
            'command.openDirectory',
            'command.getAttrs',
        ] + GLOBAL_METRICS
        makeReportService(
            c.get('sftp', 'stats_host'),
            c.getint('sftp', 'stats_port'),
            known_fields=known_fields
        ).setServiceParent(sftp_service)

    authdb = SwiftBasedAuthDB(
        c.get('sftp', 'auth_url'),
        global_max_concurrency=c.getint('sftp', 'num_persistent_connections'),
        max_concurrency=c.getint('sftp', 'num_connections_per_session'),
        timeout=c.getint('sftp', 'connection_timeout'),
        extra_headers=parse_key_value_config(c.get('sftp', 'extra_headers')),
        verbose=c.getboolean('sftp', 'verbose'),
        rewrite_scheme=c.get('sftp', 'rewrite_storage_scheme'),
        rewrite_netloc=c.get('sftp', 'rewrite_storage_netloc'),
    )

    rabbitmq_hosts = c.get('rabbitmq', 'rabbitmq_hosts')
    rabbitmq_cluster = RabbitClusterClient([RabbitReplica(host, port) \
                        for host, port in [(h,int(p)) for h,p in [r.split(':') \
                        for r in rabbitmq_hosts.split(',')]]], \
                        c.get('rabbitmq', 'username'), \
                        c.get('rabbitmq', 'password')) \
                        if rabbitmq_hosts else None
    queue_name = c.get('rabbitmq', 'queue_name')

    realm = SwftpRealm(rabbitmq_cluster, queue_name)
    sftpportal = Portal(realm)
    sftpportal.registerChecker(authdb)

    sshfactory = SSHFactory()
    protocol = SwiftSSHServerTransport
    protocol.maxConnectionsPerUser = c.getint('sftp', 'sessions_per_user')
    protocol.supportedCiphers = c.get('sftp', 'chiphers')
    protocol.supportedMACs = c.get('sftp', 'macs')
    protocol.supportedCompressions = c.get('sftp', 'compressions')
    sshfactory.protocol = protocol
    sshfactory.noisy = False
    sshfactory.portal = sftpportal
    sshfactory.services['ssh-userauth'] = SwiftSSHUserAuthServer
    sshfactory.services['ssh-connection'] = SSHConnection

    pub_key_string = file(c.get('sftp', 'pub_key')).read()
    priv_key_string = file(c.get('sftp', 'priv_key')).read()
    sshfactory.publicKeys = {
        'ssh-rsa': Key.fromString(data=pub_key_string)}
    sshfactory.privateKeys = {
        'ssh-rsa': Key.fromString(data=priv_key_string)}

    signal.signal(signal.SIGUSR1, log_runtime_info)
    signal.signal(signal.SIGUSR2, log_runtime_info)

    internet.TCPServer(
        c.getint('sftp', 'port'),
        sshfactory,
        interface=c.get('sftp', 'host')).setServiceParent(sftp_service)

    return sftp_service
示例#3
0
def makeService(options):
    """
    Makes a new swftp-ftp service. The only option is the config file
    location. See CONFIG_DEFAULTS for list of configuration options.
    """
    from twisted.protocols.ftp import FTPFactory
    from twisted.cred.portal import Portal

    from swftp.ftp.server import SwftpFTPProtocol
    from swftp.realm import SwftpRealm
    from swftp.auth import SwiftBasedAuthDB
    from swftp.utils import (
        log_runtime_info, GLOBAL_METRICS, parse_key_value_config)

    print('Starting SwFTP-ftp %s' % VERSION)

    c = get_config(options['config_file'], options)
    ftp_service = service.MultiService()

    # Add statsd service
    if c.get('ftp', 'log_statsd_host'):
        try:
            from swftp.statsd import makeService as makeStatsdService
            makeStatsdService(
                c.get('ftp', 'log_statsd_host'),
                c.getint('ftp', 'log_statsd_port'),
                sample_rate=c.getfloat('ftp', 'log_statsd_sample_rate'),
                prefix=c.get('ftp', 'log_statsd_metric_prefix')
            ).setServiceParent(ftp_service)
        except ImportError:
            sys.stderr.write('Missing Statsd Module. Requires "txstatsd" \n')

    if c.get('ftp', 'stats_host'):
        from swftp.report import makeService as makeReportService
        known_fields = [
            'command.login',
            'command.logout',
            'command.makeDirectory',
            'command.removeDirectory',
            'command.removeFile',
            'command.rename',
            'command.access',
            'command.stat',
            'command.list',
            'command.openForReading',
            'command.openForWriting',
        ] + GLOBAL_METRICS
        makeReportService(
            c.get('ftp', 'stats_host'),
            c.getint('ftp', 'stats_port'),
            known_fields=known_fields
        ).setServiceParent(ftp_service)

    authdb = SwiftBasedAuthDB(
        c.get('ftp', 'auth_url'),
        global_max_concurrency=c.getint('ftp', 'num_persistent_connections'),
        max_concurrency=c.getint('ftp', 'num_connections_per_session'),
        timeout=c.getint('ftp', 'connection_timeout'),
        extra_headers=parse_key_value_config(c.get('ftp', 'extra_headers')),
        verbose=c.getboolean('ftp', 'verbose'),
        rewrite_scheme=c.get('ftp', 'rewrite_storage_scheme'),
        rewrite_netloc=c.get('ftp', 'rewrite_storage_netloc'),
    )

    ftpportal = Portal(SwftpRealm())
    ftpportal.registerChecker(authdb)
    ftpfactory = FTPFactory(ftpportal)
    protocol = SwftpFTPProtocol
    protocol.maxConnectionsPerUser = c.getint('ftp', 'sessions_per_user')
    ftpfactory.protocol = protocol
    ftpfactory.welcomeMessage = c.get('ftp', 'welcome_message')
    ftpfactory.allowAnonymous = False
    ftpfactory.timeOut = c.getint('ftp', 'session_timeout')

    signal.signal(signal.SIGUSR1, log_runtime_info)
    signal.signal(signal.SIGUSR2, log_runtime_info)

    internet.TCPServer(
        c.getint('ftp', 'port'),
        ftpfactory,
        interface=c.get('ftp', 'host')).setServiceParent(ftp_service)
    return ftp_service
示例#4
0
def makeService(options):
    """
    Makes a new swftp-sftp service. The only option is the config file
    location. See CONFIG_DEFAULTS for list of configuration options.
    """
    from twisted.conch.ssh.factory import SSHFactory
    from twisted.conch.ssh.keys import Key
    from twisted.cred.portal import Portal

    from swftp.sftp.server import SwiftSFTPRealm, SwiftSSHServerTransport, \
        SwiftSSHConnection
    from swftp.auth import SwiftBasedAuthDB
    from swftp.utils import (
        log_runtime_info, GLOBAL_METRICS, parse_key_value_config)

    c = get_config(options['config_file'], options)

    sftp_service = service.MultiService()

    # ensure timezone is GMT
    os.environ['TZ'] = 'GMT'
    time.tzset()

    print('Starting SwFTP-sftp %s' % VERSION)

    # Add statsd service
    if c.get('sftp', 'log_statsd_host'):
        try:
            from swftp.statsd import makeService as makeStatsdService
            makeStatsdService(
                c.get('sftp', 'log_statsd_host'),
                c.getint('sftp', 'log_statsd_port'),
                sample_rate=c.getfloat('sftp', 'log_statsd_sample_rate'),
                prefix=c.get('sftp', 'log_statsd_metric_prefix')
            ).setServiceParent(sftp_service)
        except ImportError:
            sys.stderr.write('Missing Statsd Module. Requires "txstatsd" \n')

    if c.get('sftp', 'stats_host'):
        from swftp.report import makeService as makeReportService
        known_fields = [
            'command.login',
            'command.logout',
            'command.gotVersion',
            'command.openFile',
            'command.removeFile',
            'command.renameFile',
            'command.makeDirectory',
            'command.removeDirectory',
            'command.openDirectory',
            'command.getAttrs',
        ] + GLOBAL_METRICS
        makeReportService(
            c.get('sftp', 'stats_host'),
            c.getint('sftp', 'stats_port'),
            known_fields=known_fields
        ).setServiceParent(sftp_service)

    authdb = SwiftBasedAuthDB(
        c.get('sftp', 'auth_url'),
        global_max_concurrency=c.getint('sftp', 'num_persistent_connections'),
        max_concurrency=c.getint('sftp', 'num_connections_per_session'),
        timeout=c.getint('sftp', 'connection_timeout'),
        extra_headers=parse_key_value_config(c.get('sftp', 'extra_headers')),
        verbose=c.getboolean('sftp', 'verbose'))

    sftpportal = Portal(SwiftSFTPRealm())
    sftpportal.registerChecker(authdb)

    sshfactory = SSHFactory()
    sshfactory.protocol = SwiftSSHServerTransport
    sshfactory.noisy = False
    sshfactory.portal = sftpportal
    sshfactory.services['ssh-connection'] = SwiftSSHConnection

    pub_key_string = file(c.get('sftp', 'pub_key')).read()
    priv_key_string = file(c.get('sftp', 'priv_key')).read()
    sshfactory.publicKeys = {
        'ssh-rsa': Key.fromString(data=pub_key_string)}
    sshfactory.privateKeys = {
        'ssh-rsa': Key.fromString(data=priv_key_string)}

    signal.signal(signal.SIGUSR1, log_runtime_info)
    signal.signal(signal.SIGUSR2, log_runtime_info)

    internet.TCPServer(
        c.getint('sftp', 'port'),
        sshfactory,
        interface=c.get('sftp', 'host')).setServiceParent(sftp_service)

    return sftp_service
示例#5
0
def makeService(options):
    """
    Makes a new swftp-ftp service. The only option is the config file
    location. See CONFIG_DEFAULTS for list of configuration options.
    """
    from twisted.protocols.ftp import FTPFactory
    from twisted.cred.portal import Portal

    from swftp.ftp.server import SwftpFTPProtocol
    from swftp.realm import SwftpRealm
    from swftp.auth import SwiftBasedAuthDB
    from swftp.utils import (log_runtime_info, GLOBAL_METRICS,
                             parse_key_value_config)

    print('Starting SwFTP-ftp %s' % VERSION)

    c = get_config(options['config_file'], options)
    ftp_service = service.MultiService()

    # Add statsd service
    if c.get('ftp', 'log_statsd_host'):
        try:
            from swftp.statsd import makeService as makeStatsdService
            makeStatsdService(
                c.get('ftp', 'log_statsd_host'),
                c.getint('ftp', 'log_statsd_port'),
                sample_rate=c.getfloat('ftp', 'log_statsd_sample_rate'),
                prefix=c.get(
                    'ftp',
                    'log_statsd_metric_prefix')).setServiceParent(ftp_service)
        except ImportError:
            sys.stderr.write('Missing Statsd Module. Requires "txstatsd" \n')

    if c.get('ftp', 'stats_host'):
        from swftp.report import makeService as makeReportService
        known_fields = [
            'command.login',
            'command.logout',
            'command.makeDirectory',
            'command.removeDirectory',
            'command.removeFile',
            'command.rename',
            'command.access',
            'command.stat',
            'command.list',
            'command.openForReading',
            'command.openForWriting',
        ] + GLOBAL_METRICS
        makeReportService(
            c.get('ftp', 'stats_host'),
            c.getint('ftp', 'stats_port'),
            known_fields=known_fields).setServiceParent(ftp_service)

    authdb = SwiftBasedAuthDB(
        c.get('ftp', 'auth_url'),
        global_max_concurrency=c.getint('ftp', 'num_persistent_connections'),
        max_concurrency=c.getint('ftp', 'num_connections_per_session'),
        timeout=c.getint('ftp', 'connection_timeout'),
        extra_headers=parse_key_value_config(c.get('ftp', 'extra_headers')),
        verbose=c.getboolean('ftp', 'verbose'),
        rewrite_scheme=c.get('ftp', 'rewrite_storage_scheme'),
        rewrite_netloc=c.get('ftp', 'rewrite_storage_netloc'),
    )

    ftpportal = Portal(SwftpRealm())
    ftpportal.registerChecker(authdb)
    ftpfactory = FTPFactory(ftpportal)
    protocol = SwftpFTPProtocol
    protocol.maxConnectionsPerUser = c.getint('ftp', 'sessions_per_user')
    ftpfactory.protocol = protocol
    ftpfactory.welcomeMessage = c.get('ftp', 'welcome_message')
    ftpfactory.allowAnonymous = False
    ftpfactory.timeOut = c.getint('ftp', 'session_timeout')

    signal.signal(signal.SIGUSR1, log_runtime_info)
    signal.signal(signal.SIGUSR2, log_runtime_info)

    internet.TCPServer(c.getint('ftp', 'port'),
                       ftpfactory,
                       interface=c.get('ftp',
                                       'host')).setServiceParent(ftp_service)
    return ftp_service
示例#6
0
 def test_duplicate(self):
     res = parse_key_value_config('test: 1, test: 2')
     self.assertEqual(res, {'test': '2'})
示例#7
0
 def test_whitespace(self):
     res = parse_key_value_config('  test    : 1   ,   test2     :  2   ')
     self.assertEqual(res, {'test': '1', 'test2': '2'})
示例#8
0
 def test_single(self):
     res = parse_key_value_config('test: 1')
     self.assertEqual(res, {'test': '1'})
示例#9
0
 def test_empty(self):
     res = parse_key_value_config('')
     self.assertEqual(res, {})
示例#10
0
def makeService(options):
    """
    Makes a new swftp-sftp service. The only option is the config file
    location. See CONFIG_DEFAULTS for list of configuration options.
    """
    from twisted.cred.portal import Portal

    from swftp.realm import SwftpRealm
    from swftp.auth import SwiftBasedAuthDB
    from swftp.utils import (log_runtime_info, GLOBAL_METRICS,
                             parse_key_value_config)

    c = get_config(options['config_file'], options)

    smtp_service = service.MultiService()

    # ensure timezone is GMT
    os.environ['TZ'] = 'GMT'
    time.tzset()

    print('Starting SwFTP-smtp %s' % VERSION)

    authdb = SwiftBasedAuthDB(
        c.get('smtp', 'auth_url'),
        global_max_concurrency=c.getint('smtp', 'num_persistent_connections'),
        max_concurrency=c.getint('smtp', 'num_connections_per_session'),
        timeout=c.getint('smtp', 'connection_timeout'),
        extra_headers=parse_key_value_config(c.get('smtp', 'extra_headers')),
        verbose=c.getboolean('smtp', 'verbose'),
        rewrite_scheme=c.get('smtp', 'rewrite_storage_scheme'),
        rewrite_netloc=c.get('smtp', 'rewrite_storage_netloc'),
    )

    rabbitmq_hosts = c.get('rabbitmq', 'rabbitmq_hosts')
    rabbitmq_cluster = RabbitClusterClient([RabbitReplica(host, port) \
                        for host, port in [(h,int(p)) for h,p in [r.split(':') \
                        for r in rabbitmq_hosts.split(',')]]], \
                        c.get('rabbitmq', 'username'), \
                        c.get('rabbitmq', 'password')) \
                        if rabbitmq_hosts else None
    queue_name = c.get('smtp', 'queue_name')
    swift_username = c.get('smtp', 'swift_username')
    swift_password = c.get('smtp', 'swift_password')

    def swift_connect():
        d = authdb.requestAvatarId(
            UsernamePassword(swift_username, swift_password))

        def cb(connection):
            return connection

        def errback(failure):
            log.err("Swift connection failed: %s" % failure.type)
            raise SMTPServerError(451, "Internal server error")

        d.addCallbacks(cb, errback)
        return d

    @defer.inlineCallbacks
    def prepare_path():
        swift_conn = yield swift_connect()
        swift_filesystem = SwiftFileSystem(swift_conn)
        try:
            yield swift_filesystem.get_container_listing('smtp', '/')
        except swift.NotFound:
            yield swift_filesystem.makeDirectory('/smtp/')
        defer.returnValue(None)

    prepare_path()

    smtp_factory = SwftpSMTPFactory(swift_connect, rabbitmq_cluster,\
                                queue_name)

    signal.signal(signal.SIGUSR1, log_runtime_info)
    signal.signal(signal.SIGUSR2, log_runtime_info)

    internet.TCPServer(c.getint('smtp', 'port'),
                       smtp_factory,
                       interface=c.get('smtp',
                                       'host')).setServiceParent(smtp_service)

    return smtp_service
示例#11
0
 def test_whitespace(self):
     res = parse_key_value_config('  test    : 1   ,   test2     :  2   ')
     self.assertEqual(res, {'test': '1', 'test2': '2'})
示例#12
0
 def test_duplicate(self):
     res = parse_key_value_config('test: 1, test: 2')
     self.assertEqual(res, {'test': '2'})
示例#13
0
 def test_empty(self):
     res = parse_key_value_config('')
     self.assertEqual(res, {})
示例#14
0
 def test_single(self):
     res = parse_key_value_config('test: 1')
     self.assertEqual(res, {'test': '1'})