def test_two_realms_and_change_a_default(self):
fname = "container-sync-realms.conf"
fcontents = """
[DEFAULT]
mtime_check_interval = 60
[US]
key = 9ff3b71c849749dbaec4ccdd3cbab62b
cluster_dfw1 = http://dfw1.host/v1/
[UK]
key = e9569809dc8b4951accc1487aa788012
key2 = f6351bd1cc36413baa43f7ba1b45e51d
cluster_lon3 = http://lon3.host/v1/
"""
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(logger.lines_dict, {})
self.assertEqual(csr.mtime_check_interval, 60)
self.assertEqual(sorted(csr.realms()), ["UK", "US"])
self.assertEqual(csr.key("US"), "9ff3b71c849749dbaec4ccdd3cbab62b")
self.assertEqual(csr.key2("US"), None)
self.assertEqual(csr.clusters("US"), ["DFW1"])
self.assertEqual(csr.endpoint("US", "DFW1"), "http://dfw1.host/v1/")
self.assertEqual(csr.key("UK"), "e9569809dc8b4951accc1487aa788012")
self.assertEqual(csr.key2("UK"), "f6351bd1cc36413baa43f7ba1b45e51d")
self.assertEqual(csr.clusters("UK"), ["LON3"])
self.assertEqual(csr.endpoint("UK", "LON3"), "http://lon3.host/v1/")
def test_two_realms_and_change_a_default(self):
fname = 'container-sync-realms.conf'
fcontents = '''
[DEFAULT]
mtime_check_interval = 60
[US]
key = 9ff3b71c849749dbaec4ccdd3cbab62b
cluster_dfw1 = http://dfw1.host/v1/
[UK]
key = e9569809dc8b4951accc1487aa788012
key2 = f6351bd1cc36413baa43f7ba1b45e51d
cluster_lon3 = http://lon3.host/v1/
'''
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(logger.all_log_lines(), {})
self.assertEqual(csr.mtime_check_interval, 60)
self.assertEqual(sorted(csr.realms()), ['UK', 'US'])
self.assertEqual(csr.key('US'), '9ff3b71c849749dbaec4ccdd3cbab62b')
self.assertIsNone(csr.key2('US'))
self.assertEqual(csr.clusters('US'), ['DFW1'])
self.assertEqual(csr.endpoint('US', 'DFW1'),
'http://dfw1.host/v1/')
self.assertEqual(csr.key('UK'), 'e9569809dc8b4951accc1487aa788012')
self.assertEqual(csr.key2('UK'),
'f6351bd1cc36413baa43f7ba1b45e51d')
self.assertEqual(csr.clusters('UK'), ['LON3'])
self.assertEqual(csr.endpoint('UK', 'LON3'),
'http://lon3.host/v1/')
def test_two_realms_and_change_a_default(self):
fname = 'container-sync-realms.conf'
fcontents = '''
[DEFAULT]
mtime_check_interval = 60
[US]
key = 9ff3b71c849749dbaec4ccdd3cbab62b
cluster_dfw1 = http://dfw1.host/v1/
[UK]
key = e9569809dc8b4951accc1487aa788012
key2 = f6351bd1cc36413baa43f7ba1b45e51d
cluster_lon3 = http://lon3.host/v1/
'''
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(logger.all_log_lines(), {})
self.assertEqual(csr.mtime_check_interval, 60)
self.assertEqual(sorted(csr.realms()), ['UK', 'US'])
self.assertEqual(csr.key('US'), '9ff3b71c849749dbaec4ccdd3cbab62b')
self.assertEqual(csr.key2('US'), None)
self.assertEqual(csr.clusters('US'), ['DFW1'])
self.assertEqual(
csr.endpoint('US', 'DFW1'), 'http://dfw1.host/v1/')
self.assertEqual(csr.key('UK'), 'e9569809dc8b4951accc1487aa788012')
self.assertEqual(
csr.key2('UK'), 'f6351bd1cc36413baa43f7ba1b45e51d')
self.assertEqual(csr.clusters('UK'), ['LON3'])
self.assertEqual(
csr.endpoint('UK', 'LON3'), 'http://lon3.host/v1/')
def test_empty_realm(self):
fname = 'container-sync-realms.conf'
fcontents = '''
[US]
'''
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(logger.all_log_lines(), {})
self.assertEqual(csr.mtime_check_interval, 300)
self.assertEqual(csr.realms(), ['US'])
self.assertIsNone(csr.key('US'))
self.assertIsNone(csr.key2('US'))
self.assertEqual(csr.clusters('US'), [])
self.assertIsNone(csr.endpoint('US', 'JUST_TESTING'))
def test_empty_realm(self):
fname = "container-sync-realms.conf"
fcontents = """
[US]
"""
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(logger.lines_dict, {})
self.assertEqual(csr.mtime_check_interval, 300)
self.assertEqual(csr.realms(), ["US"])
self.assertEqual(csr.key("US"), None)
self.assertEqual(csr.key2("US"), None)
self.assertEqual(csr.clusters("US"), [])
self.assertEqual(csr.endpoint("US", "JUST_TESTING"), None)
def test_empty_realm(self):
fname = 'container-sync-realms.conf'
fcontents = '''
[US]
'''
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(logger.all_log_lines(), {})
self.assertEqual(csr.mtime_check_interval, 300)
self.assertEqual(csr.realms(), ['US'])
self.assertEqual(csr.key('US'), None)
self.assertEqual(csr.key2('US'), None)
self.assertEqual(csr.clusters('US'), [])
self.assertEqual(csr.endpoint('US', 'JUST_TESTING'), None)
def test_one_realm(self):
fname = "container-sync-realms.conf"
fcontents = """
[US]
key = 9ff3b71c849749dbaec4ccdd3cbab62b
cluster_dfw1 = http://dfw1.host/v1/
"""
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(logger.lines_dict, {})
self.assertEqual(csr.mtime_check_interval, 300)
self.assertEqual(csr.realms(), ["US"])
self.assertEqual(csr.key("US"), "9ff3b71c849749dbaec4ccdd3cbab62b")
self.assertEqual(csr.key2("US"), None)
self.assertEqual(csr.clusters("US"), ["DFW1"])
self.assertEqual(csr.endpoint("US", "DFW1"), "http://dfw1.host/v1/")
def test_one_realm(self):
fname = 'container-sync-realms.conf'
fcontents = '''
[US]
key = 9ff3b71c849749dbaec4ccdd3cbab62b
cluster_dfw1 = http://dfw1.host/v1/
'''
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(logger.all_log_lines(), {})
self.assertEqual(csr.mtime_check_interval, 300)
self.assertEqual(csr.realms(), ['US'])
self.assertEqual(csr.key('US'), '9ff3b71c849749dbaec4ccdd3cbab62b')
self.assertIsNone(csr.key2('US'))
self.assertEqual(csr.clusters('US'), ['DFW1'])
self.assertEqual(csr.endpoint('US', 'DFW1'),
'http://dfw1.host/v1/')
def test_one_realm(self):
fname = 'container-sync-realms.conf'
fcontents = '''
[US]
key = 9ff3b71c849749dbaec4ccdd3cbab62b
cluster_dfw1 = http://dfw1.host/v1/
'''
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(logger.all_log_lines(), {})
self.assertEqual(csr.mtime_check_interval, 300)
self.assertEqual(csr.realms(), ['US'])
self.assertEqual(csr.key('US'), '9ff3b71c849749dbaec4ccdd3cbab62b')
self.assertEqual(csr.key2('US'), None)
self.assertEqual(csr.clusters('US'), ['DFW1'])
self.assertEqual(
csr.endpoint('US', 'DFW1'), 'http://dfw1.host/v1/')
class ContainerSync(object):
"""
WSGI middleware that validates an incoming container sync request
using the container-sync-realms.conf style of container sync.
"""
def __init__(self, app, conf, logger=None):
self.app = app
self.conf = conf
self.logger = logger or get_logger(conf, log_route='container_sync')
self.realms_conf = ContainerSyncRealms(
os.path.join(conf.get('swift_dir', '/etc/swift'),
'container-sync-realms.conf'), self.logger)
self.allow_full_urls = config_true_value(
conf.get('allow_full_urls', 'true'))
# configure current realm/cluster for /info
self.realm = self.cluster = None
current = conf.get('current', None)
if current:
try:
self.realm, self.cluster = (
p.upper() for p in current.strip('/').split('/'))
except ValueError:
self.logger.error('Invalid current //REALM/CLUSTER (%s)',
current)
self.register_info()
def register_info(self):
dct = {}
for realm in self.realms_conf.realms():
clusters = self.realms_conf.clusters(realm)
if clusters:
dct[realm] = {'clusters': dict((c, {}) for c in clusters)}
if self.realm and self.cluster:
try:
dct[self.realm]['clusters'][self.cluster]['current'] = True
except KeyError:
self.logger.error('Unknown current //REALM/CLUSTER (%s)',
'//%s/%s' % (self.realm, self.cluster))
register_swift_info('container_sync', realms=dct)
@wsgify
def __call__(self, req):
if req.path == '/info':
# Ensure /info requests get the freshest results
self.register_info()
return self.app
try:
(version, acc, cont, obj) = req.split_path(3, 4, True)
bad_path = False
except ValueError:
bad_path = True
# use of bad_path bool is to avoid recursive tracebacks
if bad_path or not valid_api_version(version):
return self.app
# validate container-sync metdata update
info = get_container_info(req.environ, self.app, swift_source='CS')
sync_to = req.headers.get('x-container-sync-to')
if req.method in ('PUT', 'POST') and cont and not obj:
versions_cont = info.get('sysmeta', {}).get('versions-container')
if sync_to and versions_cont:
raise HTTPBadRequest(
'Cannot configure container sync on a container '
'with object versioning configured.',
request=req)
if not self.allow_full_urls:
if sync_to and not sync_to.startswith('//'):
raise HTTPBadRequest(
body='Full URLs are not allowed for X-Container-Sync-To '
'values. Only realm values of the format '
'//realm/cluster/account/container are allowed.\n',
request=req)
auth = req.headers.get('x-container-sync-auth')
if auth:
valid = False
auth = auth.split()
if len(auth) != 3:
req.environ.setdefault('swift.log_info',
[]).append('cs:not-3-args')
else:
realm, nonce, sig = auth
realm_key = self.realms_conf.key(realm)
realm_key2 = self.realms_conf.key2(realm)
if not realm_key:
req.environ.setdefault('swift.log_info',
[]).append('cs:no-local-realm-key')
else:
user_key = info.get('sync_key')
if not user_key:
req.environ.setdefault(
'swift.log_info',
[]).append('cs:no-local-user-key')
else:
# x-timestamp headers get shunted by gatekeeper
if 'x-backend-inbound-x-timestamp' in req.headers:
req.headers['x-timestamp'] = req.headers.pop(
'x-backend-inbound-x-timestamp')
expected = self.realms_conf.get_sig(
req.method, req.path,
req.headers.get('x-timestamp', '0'), nonce,
realm_key, user_key)
expected2 = self.realms_conf.get_sig(
req.method, req.path,
req.headers.get('x-timestamp', '0'), nonce,
realm_key2, user_key) if realm_key2 else expected
if not streq_const_time(sig, expected) and \
not streq_const_time(sig, expected2):
req.environ.setdefault('swift.log_info',
[]).append('cs:invalid-sig')
else:
req.environ.setdefault('swift.log_info',
[]).append('cs:valid')
valid = True
if not valid:
exc = HTTPUnauthorized(
body='X-Container-Sync-Auth header not valid; '
'contact cluster operator for support.',
headers={'content-type': 'text/plain'},
request=req)
exc.headers['www-authenticate'] = ' '.join([
'SwiftContainerSync',
exc.www_authenticate().split(None, 1)[1]
])
raise exc
else:
req.environ['swift.authorize_override'] = True
# An SLO manifest will already be in the internal manifest
# syntax and might be synced before its segments, so stop SLO
# middleware from performing the usual manifest validation.
req.environ['swift.slo_override'] = True
# Similar arguments for static symlinks
req.environ['swift.symlink_override'] = True
return self.app
class ContainerSync(object):
"""
WSGI middleware that validates an incoming container sync request
using the container-sync-realms.conf style of container sync.
"""
def __init__(self, app, conf):
self.app = app
self.conf = conf
self.logger = get_logger(conf, log_route='container_sync')
self.realms_conf = ContainerSyncRealms(
os.path.join(conf.get('swift_dir', '/etc/swift'),
'container-sync-realms.conf'), self.logger)
self.allow_full_urls = config_true_value(
conf.get('allow_full_urls', 'true'))
@wsgify
def __call__(self, req):
if not self.allow_full_urls:
sync_to = req.headers.get('x-container-sync-to')
if sync_to and not sync_to.startswith('//'):
raise HTTPBadRequest(
body='Full URLs are not allowed for X-Container-Sync-To '
'values. Only realm values of the format '
'//realm/cluster/account/container are allowed.\n',
request=req)
auth = req.headers.get('x-container-sync-auth')
if auth:
valid = False
auth = auth.split()
if len(auth) != 3:
req.environ.setdefault('swift.log_info',
[]).append('cs:not-3-args')
else:
realm, nonce, sig = auth
realm_key = self.realms_conf.key(realm)
realm_key2 = self.realms_conf.key2(realm)
if not realm_key:
req.environ.setdefault('swift.log_info',
[]).append('cs:no-local-realm-key')
else:
info = get_container_info(req.environ,
self.app,
swift_source='CS')
user_key = info.get('sync_key')
if not user_key:
req.environ.setdefault(
'swift.log_info',
[]).append('cs:no-local-user-key')
else:
expected = self.realms_conf.get_sig(
req.method, req.path,
req.headers.get('x-timestamp', '0'), nonce,
realm_key, user_key)
expected2 = self.realms_conf.get_sig(
req.method, req.path,
req.headers.get('x-timestamp', '0'), nonce,
realm_key2, user_key) if realm_key2 else expected
if not streq_const_time(sig, expected) and \
not streq_const_time(sig, expected2):
req.environ.setdefault('swift.log_info',
[]).append('cs:invalid-sig')
else:
req.environ.setdefault('swift.log_info',
[]).append('cs:valid')
valid = True
if not valid:
exc = HTTPUnauthorized(
body='X-Container-Sync-Auth header not valid; '
'contact cluster operator for support.',
headers={'content-type': 'text/plain'},
request=req)
exc.headers['www-authenticate'] = ' '.join([
'SwiftContainerSync',
exc.www_authenticate().split(None, 1)[1]
])
raise exc
else:
req.environ['swift.authorize_override'] = True
if req.path == '/info':
# Ensure /info requests get the freshest results
dct = {}
for realm in self.realms_conf.realms():
clusters = self.realms_conf.clusters(realm)
if clusters:
dct[realm] = {'clusters': dict((c, {}) for c in clusters)}
register_swift_info('container_sync', realms=dct)
return self.app
class ContainerSync(object):
"""
WSGI middleware that validates an incoming container sync request
using the container-sync-realms.conf style of container sync.
"""
def __init__(self, app, conf):
self.app = app
self.conf = conf
self.logger = get_logger(conf, log_route='container_sync')
self.realms_conf = ContainerSyncRealms(
os.path.join(
conf.get('swift_dir', '/etc/swift'),
'container-sync-realms.conf'),
self.logger)
self.allow_full_urls = config_true_value(
conf.get('allow_full_urls', 'true'))
@wsgify
def __call__(self, req):
if not self.allow_full_urls:
sync_to = req.headers.get('x-container-sync-to')
if sync_to and not sync_to.startswith('//'):
raise HTTPBadRequest(
body='Full URLs are not allowed for X-Container-Sync-To '
'values. Only realm values of the format '
'//realm/cluster/account/container are allowed.\n',
request=req)
auth = req.headers.get('x-container-sync-auth')
if auth:
valid = False
auth = auth.split()
if len(auth) != 3:
req.environ.setdefault('swift.log_info', []).append(
'cs:not-3-args')
else:
realm, nonce, sig = auth
realm_key = self.realms_conf.key(realm)
realm_key2 = self.realms_conf.key2(realm)
if not realm_key:
req.environ.setdefault('swift.log_info', []).append(
'cs:no-local-realm-key')
else:
info = get_container_info(
req.environ, self.app, swift_source='CS')
user_key = info.get('sync_key')
if not user_key:
req.environ.setdefault('swift.log_info', []).append(
'cs:no-local-user-key')
else:
expected = self.realms_conf.get_sig(
req.method, req.path,
req.headers.get('x-timestamp', '0'), nonce,
realm_key, user_key)
expected2 = self.realms_conf.get_sig(
req.method, req.path,
req.headers.get('x-timestamp', '0'), nonce,
realm_key2, user_key) if realm_key2 else expected
if not streq_const_time(sig, expected) and \
not streq_const_time(sig, expected2):
req.environ.setdefault(
'swift.log_info', []).append('cs:invalid-sig')
else:
req.environ.setdefault(
'swift.log_info', []).append('cs:valid')
valid = True
if not valid:
exc = HTTPUnauthorized(
body='X-Container-Sync-Auth header not valid; '
'contact cluster operator for support.',
headers={'content-type': 'text/plain'},
request=req)
exc.headers['www-authenticate'] = ' '.join([
'SwiftContainerSync',
exc.www_authenticate().split(None, 1)[1]])
raise exc
else:
req.environ['swift.authorize_override'] = True
if req.path == '/info':
# Ensure /info requests get the freshest results
dct = {}
for realm in self.realms_conf.realms():
clusters = self.realms_conf.clusters(realm)
if clusters:
dct[realm] = {'clusters': dict((c, {}) for c in clusters)}
register_swift_info('container_sync', realms=dct)
return self.app
class ContainerSync(object):
"""
WSGI middleware that validates an incoming container sync request
using the container-sync-realms.conf style of container sync.
"""
def __init__(self, app, conf, logger=None):
self.app = app
self.conf = conf
self.logger = logger or get_logger(conf, log_route='container_sync')
self.realms_conf = ContainerSyncRealms(
os.path.join(
conf.get('swift_dir', '/etc/swift'),
'container-sync-realms.conf'),
self.logger)
self.allow_full_urls = config_true_value(
conf.get('allow_full_urls', 'true'))
# configure current realm/cluster for /info
self.realm = self.cluster = None
current = conf.get('current', None)
if current:
try:
self.realm, self.cluster = (p.upper() for p in
current.strip('/').split('/'))
except ValueError:
self.logger.error('Invalid current //REALM/CLUSTER (%s)',
current)
self.register_info()
def register_info(self):
dct = {}
for realm in self.realms_conf.realms():
clusters = self.realms_conf.clusters(realm)
if clusters:
dct[realm] = {'clusters': dict((c, {}) for c in clusters)}
if self.realm and self.cluster:
try:
dct[self.realm]['clusters'][self.cluster]['current'] = True
except KeyError:
self.logger.error('Unknown current //REALM/CLUSTER (%s)',
'//%s/%s' % (self.realm, self.cluster))
register_swift_info('container_sync', realms=dct)
@wsgify
def __call__(self, req):
if not self.allow_full_urls:
sync_to = req.headers.get('x-container-sync-to')
if sync_to and not sync_to.startswith('//'):
raise HTTPBadRequest(
body='Full URLs are not allowed for X-Container-Sync-To '
'values. Only realm values of the format '
'//realm/cluster/account/container are allowed.\n',
request=req)
auth = req.headers.get('x-container-sync-auth')
if auth:
valid = False
auth = auth.split()
if len(auth) != 3:
req.environ.setdefault('swift.log_info', []).append(
'cs:not-3-args')
else:
realm, nonce, sig = auth
realm_key = self.realms_conf.key(realm)
realm_key2 = self.realms_conf.key2(realm)
if not realm_key:
req.environ.setdefault('swift.log_info', []).append(
'cs:no-local-realm-key')
else:
info = get_container_info(
req.environ, self.app, swift_source='CS')
user_key = info.get('sync_key')
if not user_key:
req.environ.setdefault('swift.log_info', []).append(
'cs:no-local-user-key')
else:
# x-timestamp headers get shunted by gatekeeper
if 'x-backend-inbound-x-timestamp' in req.headers:
req.headers['x-timestamp'] = req.headers.pop(
'x-backend-inbound-x-timestamp')
expected = self.realms_conf.get_sig(
req.method, req.path,
req.headers.get('x-timestamp', '0'), nonce,
realm_key, user_key)
expected2 = self.realms_conf.get_sig(
req.method, req.path,
req.headers.get('x-timestamp', '0'), nonce,
realm_key2, user_key) if realm_key2 else expected
if not streq_const_time(sig, expected) and \
not streq_const_time(sig, expected2):
req.environ.setdefault(
'swift.log_info', []).append('cs:invalid-sig')
else:
req.environ.setdefault(
'swift.log_info', []).append('cs:valid')
valid = True
if not valid:
exc = HTTPUnauthorized(
body='X-Container-Sync-Auth header not valid; '
'contact cluster operator for support.',
headers={'content-type': 'text/plain'},
request=req)
exc.headers['www-authenticate'] = ' '.join([
'SwiftContainerSync',
exc.www_authenticate().split(None, 1)[1]])
raise exc
else:
req.environ['swift.authorize_override'] = True
# An SLO manifest will already be in the internal manifest
# syntax and might be synced before its segments, so stop SLO
# middleware from performing the usual manifest validation.
req.environ['swift.slo_override'] = True
if req.path == '/info':
# Ensure /info requests get the freshest results
self.register_info()
return self.app
