def makeService(options):
"""
Makes a new swftp-sftp service. The only option is the config file
location. See CONFIG_DEFAULTS for list of configuration options.
"""
from twisted.cred.portal import Portal
from swftp.realm import SwftpRealm
from swftp.auth import SwiftBasedAuthDB
from swftp.utils import (
log_runtime_info, GLOBAL_METRICS, parse_key_value_config)
c = get_config(options['config_file'], options)
smtp_service = service.MultiService()
# ensure timezone is GMT
os.environ['TZ'] = 'GMT'
time.tzset()
print('Starting SwFTP-smtp %s' % VERSION)
authdb = SwiftBasedAuthDB(
c.get('smtp', 'auth_url'),
global_max_concurrency=c.getint('smtp', 'num_persistent_connections'),
max_concurrency=c.getint('smtp', 'num_connections_per_session'),
timeout=c.getint('smtp', 'connection_timeout'),
extra_headers=parse_key_value_config(c.get('smtp', 'extra_headers')),
verbose=c.getboolean('smtp', 'verbose'),
rewrite_scheme=c.get('smtp', 'rewrite_storage_scheme'),
rewrite_netloc=c.get('smtp', 'rewrite_storage_netloc'),
)
rabbitmq_hosts = c.get('rabbitmq', 'rabbitmq_hosts')
rabbitmq_cluster = RabbitClusterClient([RabbitReplica(host, port) \
for host, port in [(h,int(p)) for h,p in [r.split(':') \
for r in rabbitmq_hosts.split(',')]]], \
c.get('rabbitmq', 'username'), \
c.get('rabbitmq', 'password')) \
if rabbitmq_hosts else None
queue_name = c.get('smtp', 'queue_name')
swift_username = c.get('smtp', 'swift_username')
swift_password = c.get('smtp', 'swift_password')
def swift_connect():
d = authdb.requestAvatarId(UsernamePassword(swift_username, swift_password))
def cb(connection):
return connection
def errback(failure):
log.err("Swift connection failed: %s" % failure.type)
raise SMTPServerError(451, "Internal server error")
d.addCallbacks(cb, errback)
return d
@defer.inlineCallbacks
def prepare_path():
swift_conn = yield swift_connect()
swift_filesystem = SwiftFileSystem(swift_conn)
try:
yield swift_filesystem.get_container_listing('smtp', '/')
except swift.NotFound:
yield swift_filesystem.makeDirectory('/smtp/')
defer.returnValue(None)
prepare_path()
smtp_factory = SwftpSMTPFactory(swift_connect, rabbitmq_cluster,\
queue_name)
signal.signal(signal.SIGUSR1, log_runtime_info)
signal.signal(signal.SIGUSR2, log_runtime_info)
internet.TCPServer(
c.getint('smtp', 'port'),
smtp_factory,
interface=c.get('smtp', 'host')).setServiceParent(smtp_service)
return smtp_service
def makeService(options):
"""
Makes a new swftp-sftp service. The only option is the config file
location. See CONFIG_DEFAULTS for list of configuration options.
"""
from twisted.conch.ssh.connection import SSHConnection
from twisted.conch.ssh.factory import SSHFactory
from twisted.conch.ssh.keys import Key
from twisted.cred.portal import Portal
from swftp.realm import SwftpRealm
from swftp.sftp.server import SwiftSSHUserAuthServer
from swftp.auth import SwiftBasedAuthDB
from swftp.utils import (
log_runtime_info, GLOBAL_METRICS, parse_key_value_config)
c = get_config(options['config_file'], options)
sftp_service = service.MultiService()
# ensure timezone is GMT
os.environ['TZ'] = 'GMT'
time.tzset()
print('Starting SwFTP-sftp %s' % VERSION)
# Add statsd service
if c.get('sftp', 'log_statsd_host'):
try:
from swftp.statsd import makeService as makeStatsdService
makeStatsdService(
c.get('sftp', 'log_statsd_host'),
c.getint('sftp', 'log_statsd_port'),
sample_rate=c.getfloat('sftp', 'log_statsd_sample_rate'),
prefix=c.get('sftp', 'log_statsd_metric_prefix')
).setServiceParent(sftp_service)
except ImportError:
sys.stderr.write('Missing Statsd Module. Requires "txstatsd" \n')
if c.get('sftp', 'stats_host'):
from swftp.report import makeService as makeReportService
known_fields = [
'command.login',
'command.logout',
'command.gotVersion',
'command.openFile',
'command.removeFile',
'command.renameFile',
'command.makeDirectory',
'command.removeDirectory',
'command.openDirectory',
'command.getAttrs',
] + GLOBAL_METRICS
makeReportService(
c.get('sftp', 'stats_host'),
c.getint('sftp', 'stats_port'),
known_fields=known_fields
).setServiceParent(sftp_service)
authdb = SwiftBasedAuthDB(
c.get('sftp', 'auth_url'),
global_max_concurrency=c.getint('sftp', 'num_persistent_connections'),
max_concurrency=c.getint('sftp', 'num_connections_per_session'),
timeout=c.getint('sftp', 'connection_timeout'),
extra_headers=parse_key_value_config(c.get('sftp', 'extra_headers')),
verbose=c.getboolean('sftp', 'verbose'),
rewrite_scheme=c.get('sftp', 'rewrite_storage_scheme'),
rewrite_netloc=c.get('sftp', 'rewrite_storage_netloc'),
)
rabbitmq_hosts = c.get('rabbitmq', 'rabbitmq_hosts')
rabbitmq_cluster = RabbitClusterClient([RabbitReplica(host, port) \
for host, port in [(h,int(p)) for h,p in [r.split(':') \
for r in rabbitmq_hosts.split(',')]]], \
c.get('rabbitmq', 'username'), \
c.get('rabbitmq', 'password')) \
if rabbitmq_hosts else None
queue_name = c.get('rabbitmq', 'queue_name')
realm = SwftpRealm(rabbitmq_cluster, queue_name)
sftpportal = Portal(realm)
sftpportal.registerChecker(authdb)
sshfactory = SSHFactory()
protocol = SwiftSSHServerTransport
protocol.maxConnectionsPerUser = c.getint('sftp', 'sessions_per_user')
protocol.supportedCiphers = c.get('sftp', 'chiphers')
protocol.supportedMACs = c.get('sftp', 'macs')
protocol.supportedCompressions = c.get('sftp', 'compressions')
sshfactory.protocol = protocol
sshfactory.noisy = False
sshfactory.portal = sftpportal
sshfactory.services['ssh-userauth'] = SwiftSSHUserAuthServer
sshfactory.services['ssh-connection'] = SSHConnection
pub_key_string = file(c.get('sftp', 'pub_key')).read()
priv_key_string = file(c.get('sftp', 'priv_key')).read()
sshfactory.publicKeys = {
'ssh-rsa': Key.fromString(data=pub_key_string)}
sshfactory.privateKeys = {
'ssh-rsa': Key.fromString(data=priv_key_string)}
signal.signal(signal.SIGUSR1, log_runtime_info)
signal.signal(signal.SIGUSR2, log_runtime_info)
internet.TCPServer(
c.getint('sftp', 'port'),
sshfactory,
interface=c.get('sftp', 'host')).setServiceParent(sftp_service)
return sftp_service
def makeService(options):
"""
Makes a new swftp-ftp service. The only option is the config file
location. See CONFIG_DEFAULTS for list of configuration options.
"""
from twisted.protocols.ftp import FTPFactory
from twisted.cred.portal import Portal
from swftp.ftp.server import SwftpFTPProtocol
from swftp.realm import SwftpRealm
from swftp.auth import SwiftBasedAuthDB
from swftp.utils import (
log_runtime_info, GLOBAL_METRICS, parse_key_value_config)
print('Starting SwFTP-ftp %s' % VERSION)
c = get_config(options['config_file'], options)
ftp_service = service.MultiService()
# Add statsd service
if c.get('ftp', 'log_statsd_host'):
try:
from swftp.statsd import makeService as makeStatsdService
makeStatsdService(
c.get('ftp', 'log_statsd_host'),
c.getint('ftp', 'log_statsd_port'),
sample_rate=c.getfloat('ftp', 'log_statsd_sample_rate'),
prefix=c.get('ftp', 'log_statsd_metric_prefix')
).setServiceParent(ftp_service)
except ImportError:
sys.stderr.write('Missing Statsd Module. Requires "txstatsd" \n')
if c.get('ftp', 'stats_host'):
from swftp.report import makeService as makeReportService
known_fields = [
'command.login',
'command.logout',
'command.makeDirectory',
'command.removeDirectory',
'command.removeFile',
'command.rename',
'command.access',
'command.stat',
'command.list',
'command.openForReading',
'command.openForWriting',
] + GLOBAL_METRICS
makeReportService(
c.get('ftp', 'stats_host'),
c.getint('ftp', 'stats_port'),
known_fields=known_fields
).setServiceParent(ftp_service)
authdb = SwiftBasedAuthDB(
c.get('ftp', 'auth_url'),
global_max_concurrency=c.getint('ftp', 'num_persistent_connections'),
max_concurrency=c.getint('ftp', 'num_connections_per_session'),
timeout=c.getint('ftp', 'connection_timeout'),
extra_headers=parse_key_value_config(c.get('ftp', 'extra_headers')),
verbose=c.getboolean('ftp', 'verbose'),
rewrite_scheme=c.get('ftp', 'rewrite_storage_scheme'),
rewrite_netloc=c.get('ftp', 'rewrite_storage_netloc'),
)
ftpportal = Portal(SwftpRealm())
ftpportal.registerChecker(authdb)
ftpfactory = FTPFactory(ftpportal)
protocol = SwftpFTPProtocol
protocol.maxConnectionsPerUser = c.getint('ftp', 'sessions_per_user')
ftpfactory.protocol = protocol
ftpfactory.welcomeMessage = c.get('ftp', 'welcome_message')
ftpfactory.allowAnonymous = False
ftpfactory.timeOut = c.getint('ftp', 'session_timeout')
signal.signal(signal.SIGUSR1, log_runtime_info)
signal.signal(signal.SIGUSR2, log_runtime_info)
internet.TCPServer(
c.getint('ftp', 'port'),
ftpfactory,
interface=c.get('ftp', 'host')).setServiceParent(ftp_service)
return ftp_service
def makeService(options):
"""
Makes a new swftp-sftp service. The only option is the config file
location. See CONFIG_DEFAULTS for list of configuration options.
"""
from twisted.conch.ssh.factory import SSHFactory
from twisted.conch.ssh.keys import Key
from twisted.cred.portal import Portal
from swftp.sftp.server import SwiftSFTPRealm, SwiftSSHServerTransport, \
SwiftSSHConnection
from swftp.auth import SwiftBasedAuthDB
from swftp.utils import (
log_runtime_info, GLOBAL_METRICS, parse_key_value_config)
c = get_config(options['config_file'], options)
sftp_service = service.MultiService()
# ensure timezone is GMT
os.environ['TZ'] = 'GMT'
time.tzset()
print('Starting SwFTP-sftp %s' % VERSION)
# Add statsd service
if c.get('sftp', 'log_statsd_host'):
try:
from swftp.statsd import makeService as makeStatsdService
makeStatsdService(
c.get('sftp', 'log_statsd_host'),
c.getint('sftp', 'log_statsd_port'),
sample_rate=c.getfloat('sftp', 'log_statsd_sample_rate'),
prefix=c.get('sftp', 'log_statsd_metric_prefix')
).setServiceParent(sftp_service)
except ImportError:
sys.stderr.write('Missing Statsd Module. Requires "txstatsd" \n')
if c.get('sftp', 'stats_host'):
from swftp.report import makeService as makeReportService
known_fields = [
'command.login',
'command.logout',
'command.gotVersion',
'command.openFile',
'command.removeFile',
'command.renameFile',
'command.makeDirectory',
'command.removeDirectory',
'command.openDirectory',
'command.getAttrs',
] + GLOBAL_METRICS
makeReportService(
c.get('sftp', 'stats_host'),
c.getint('sftp', 'stats_port'),
known_fields=known_fields
).setServiceParent(sftp_service)
authdb = SwiftBasedAuthDB(
c.get('sftp', 'auth_url'),
global_max_concurrency=c.getint('sftp', 'num_persistent_connections'),
max_concurrency=c.getint('sftp', 'num_connections_per_session'),
timeout=c.getint('sftp', 'connection_timeout'),
extra_headers=parse_key_value_config(c.get('sftp', 'extra_headers')),
verbose=c.getboolean('sftp', 'verbose'))
sftpportal = Portal(SwiftSFTPRealm())
sftpportal.registerChecker(authdb)
sshfactory = SSHFactory()
sshfactory.protocol = SwiftSSHServerTransport
sshfactory.noisy = False
sshfactory.portal = sftpportal
sshfactory.services['ssh-connection'] = SwiftSSHConnection
pub_key_string = file(c.get('sftp', 'pub_key')).read()
priv_key_string = file(c.get('sftp', 'priv_key')).read()
sshfactory.publicKeys = {
'ssh-rsa': Key.fromString(data=pub_key_string)}
sshfactory.privateKeys = {
'ssh-rsa': Key.fromString(data=priv_key_string)}
signal.signal(signal.SIGUSR1, log_runtime_info)
signal.signal(signal.SIGUSR2, log_runtime_info)
internet.TCPServer(
c.getint('sftp', 'port'),
sshfactory,
interface=c.get('sftp', 'host')).setServiceParent(sftp_service)
return sftp_service
def makeService(options):
"""
Makes a new swftp-ftp service. The only option is the config file
location. See CONFIG_DEFAULTS for list of configuration options.
"""
from twisted.protocols.ftp import FTPFactory
from twisted.cred.portal import Portal
from swftp.ftp.server import SwftpFTPProtocol
from swftp.realm import SwftpRealm
from swftp.auth import SwiftBasedAuthDB
from swftp.utils import (log_runtime_info, GLOBAL_METRICS,
parse_key_value_config)
print('Starting SwFTP-ftp %s' % VERSION)
c = get_config(options['config_file'], options)
ftp_service = service.MultiService()
# Add statsd service
if c.get('ftp', 'log_statsd_host'):
try:
from swftp.statsd import makeService as makeStatsdService
makeStatsdService(
c.get('ftp', 'log_statsd_host'),
c.getint('ftp', 'log_statsd_port'),
sample_rate=c.getfloat('ftp', 'log_statsd_sample_rate'),
prefix=c.get(
'ftp',
'log_statsd_metric_prefix')).setServiceParent(ftp_service)
except ImportError:
sys.stderr.write('Missing Statsd Module. Requires "txstatsd" \n')
if c.get('ftp', 'stats_host'):
from swftp.report import makeService as makeReportService
known_fields = [
'command.login',
'command.logout',
'command.makeDirectory',
'command.removeDirectory',
'command.removeFile',
'command.rename',
'command.access',
'command.stat',
'command.list',
'command.openForReading',
'command.openForWriting',
] + GLOBAL_METRICS
makeReportService(
c.get('ftp', 'stats_host'),
c.getint('ftp', 'stats_port'),
known_fields=known_fields).setServiceParent(ftp_service)
authdb = SwiftBasedAuthDB(
c.get('ftp', 'auth_url'),
global_max_concurrency=c.getint('ftp', 'num_persistent_connections'),
max_concurrency=c.getint('ftp', 'num_connections_per_session'),
timeout=c.getint('ftp', 'connection_timeout'),
extra_headers=parse_key_value_config(c.get('ftp', 'extra_headers')),
verbose=c.getboolean('ftp', 'verbose'),
rewrite_scheme=c.get('ftp', 'rewrite_storage_scheme'),
rewrite_netloc=c.get('ftp', 'rewrite_storage_netloc'),
)
ftpportal = Portal(SwftpRealm())
ftpportal.registerChecker(authdb)
ftpfactory = FTPFactory(ftpportal)
protocol = SwftpFTPProtocol
protocol.maxConnectionsPerUser = c.getint('ftp', 'sessions_per_user')
ftpfactory.protocol = protocol
ftpfactory.welcomeMessage = c.get('ftp', 'welcome_message')
ftpfactory.allowAnonymous = False
ftpfactory.timeOut = c.getint('ftp', 'session_timeout')
signal.signal(signal.SIGUSR1, log_runtime_info)
signal.signal(signal.SIGUSR2, log_runtime_info)
internet.TCPServer(c.getint('ftp', 'port'),
ftpfactory,
interface=c.get('ftp',
'host')).setServiceParent(ftp_service)
return ftp_service
def test_duplicate(self):
res = parse_key_value_config('test: 1, test: 2')
self.assertEqual(res, {'test': '2'})
def test_whitespace(self):
res = parse_key_value_config(' test : 1 , test2 : 2 ')
self.assertEqual(res, {'test': '1', 'test2': '2'})
def test_single(self):
res = parse_key_value_config('test: 1')
self.assertEqual(res, {'test': '1'})
def test_empty(self):
res = parse_key_value_config('')
self.assertEqual(res, {})
def makeService(options):
"""
Makes a new swftp-sftp service. The only option is the config file
location. See CONFIG_DEFAULTS for list of configuration options.
"""
from twisted.cred.portal import Portal
from swftp.realm import SwftpRealm
from swftp.auth import SwiftBasedAuthDB
from swftp.utils import (log_runtime_info, GLOBAL_METRICS,
parse_key_value_config)
c = get_config(options['config_file'], options)
smtp_service = service.MultiService()
# ensure timezone is GMT
os.environ['TZ'] = 'GMT'
time.tzset()
print('Starting SwFTP-smtp %s' % VERSION)
authdb = SwiftBasedAuthDB(
c.get('smtp', 'auth_url'),
global_max_concurrency=c.getint('smtp', 'num_persistent_connections'),
max_concurrency=c.getint('smtp', 'num_connections_per_session'),
timeout=c.getint('smtp', 'connection_timeout'),
extra_headers=parse_key_value_config(c.get('smtp', 'extra_headers')),
verbose=c.getboolean('smtp', 'verbose'),
rewrite_scheme=c.get('smtp', 'rewrite_storage_scheme'),
rewrite_netloc=c.get('smtp', 'rewrite_storage_netloc'),
)
rabbitmq_hosts = c.get('rabbitmq', 'rabbitmq_hosts')
rabbitmq_cluster = RabbitClusterClient([RabbitReplica(host, port) \
for host, port in [(h,int(p)) for h,p in [r.split(':') \
for r in rabbitmq_hosts.split(',')]]], \
c.get('rabbitmq', 'username'), \
c.get('rabbitmq', 'password')) \
if rabbitmq_hosts else None
queue_name = c.get('smtp', 'queue_name')
swift_username = c.get('smtp', 'swift_username')
swift_password = c.get('smtp', 'swift_password')
def swift_connect():
d = authdb.requestAvatarId(
UsernamePassword(swift_username, swift_password))
def cb(connection):
return connection
def errback(failure):
log.err("Swift connection failed: %s" % failure.type)
raise SMTPServerError(451, "Internal server error")
d.addCallbacks(cb, errback)
return d
@defer.inlineCallbacks
def prepare_path():
swift_conn = yield swift_connect()
swift_filesystem = SwiftFileSystem(swift_conn)
try:
yield swift_filesystem.get_container_listing('smtp', '/')
except swift.NotFound:
yield swift_filesystem.makeDirectory('/smtp/')
defer.returnValue(None)
prepare_path()
smtp_factory = SwftpSMTPFactory(swift_connect, rabbitmq_cluster,\
queue_name)
signal.signal(signal.SIGUSR1, log_runtime_info)
signal.signal(signal.SIGUSR2, log_runtime_info)
internet.TCPServer(c.getint('smtp', 'port'),
smtp_factory,
interface=c.get('smtp',
'host')).setServiceParent(smtp_service)
return smtp_service
def test_whitespace(self):
res = parse_key_value_config(' test : 1 , test2 : 2 ')
self.assertEqual(res, {'test': '1', 'test2': '2'})
def test_duplicate(self):
res = parse_key_value_config('test: 1, test: 2')
self.assertEqual(res, {'test': '2'})
def test_empty(self):
res = parse_key_value_config('')
self.assertEqual(res, {})
def test_single(self):
res = parse_key_value_config('test: 1')
self.assertEqual(res, {'test': '1'})