def test_validate_ring(self):
self._setup_obj_ring()
#ok ring and ok md5
obj_ring = os.path.join(self.testdir, 'object.ring.gz')
obj_ring_md5 = get_md5sum(obj_ring)
minion = RingMinion(conf={'swiftdir': self.testdir})
minion._validate_ring(obj_ring, obj_ring_md5)
# m5 miss match
try:
minion._validate_ring(obj_ring, 'badmd5')
except Exception as err:
self.assertEqual(err.message, "md5 missmatch")
else:
self.fail('Should have thrown md5 missmatch exception')
# bad ring file
bfile = os.path.join(self.testdir, 'test.ring.gz')
with open(bfile, 'w') as f:
f.write('whatisthis.')
test_md5 = get_md5sum(bfile)
try:
minion._validate_ring(bfile, test_md5)
except Exception as err:
self.assertEqual(err.message, "Invalid ring")
else:
self.fail('Should have thrown Invalid ring exception')
def _validate_file(self, filename):
"""Validate md5 of file"""
if self._changed(filename):
self.logger.debug("updating md5")
with lock_parent_directory(self.swiftdir, self.lock_timeout):
self.last_tstamp[filename] = stat(filename).st_mtime
self.current_md5[filename] = get_md5sum(filename)
def __init__(self, conf):
self.current_md5 = {}
self.swiftdir = conf.get('swiftdir', '/etc/swift')
self.rings = {'account': conf.get('account_ring',
pathjoin(self.swiftdir,
'account.ring.gz')),
'container': conf.get('container_ring',
pathjoin(self.swiftdir,
'container.ring.gz')),
'object': conf.get('object_ring',
pathjoin(self.swiftdir,
'object.ring.gz'))}
self.start_delay = int(conf.get('start_delay_range', '120'))
self.check_interval = int(conf.get('check_interval', '30'))
self.ring_master = conf.get('ring_master', 'http://127.0.0.1:8090/')
self.ring_master_timeout = int(conf.get('ring_master_timeout', '30'))
self.debug = conf.get('debug', 'n') in TRUE_VALUES
if self.debug:
conf['log_level'] = 'DEBUG'
self.logger = get_logger(conf, 'ringminiond', self.debug)
if not os.access(self.swiftdir, os.W_OK):
self.logger.error('swift_dir is not writable. exiting!')
sys.exit(1)
for ring in self.rings:
if exists(self.rings[ring]):
self.current_md5[self.rings[ring]] = \
get_md5sum(self.rings[ring])
else:
self.current_md5[self.rings[ring]] = ''
def write_ring(self, btype, builder):
"""Write out new ring files
:param btype: The builder type
:param builder: The builder to dump
:returns: new ring file md5
"""
try:
self.pause_if_asked()
ring_file = self.ring_files[btype]
fd, tmppath = mkstemp(dir=self.swiftdir, suffix='.tmp.ring.gz')
builder.get_ring().save(tmppath)
close(fd)
if not is_valid_ring(tmppath):
unlink(tmppath)
raise Exception('Ring Validate Failed')
backup, backup_md5 = make_backup(ring_file, self.backup_dir)
self.logger.notice('--> Backed up %s to %s (%s)' %
(ring_file, backup, backup_md5))
chmod(tmppath, 0644)
rename(tmppath, ring_file)
except Exception as err:
raise Exception('Error writing builder: %s' % err)
finally:
if fd:
try:
close(fd)
except OSError:
pass
if tmppath:
try:
unlink(tmppath)
except OSError:
pass
return get_md5sum(ring_file)
def write_builder(self, btype, builder):
"""Write out new builder file
:param btype: The builder type
:param builder: The builder to dump
:returns: new ring file md5
"""
self.pause_if_asked()
builder_file = self.builder_files[btype]
try:
fd, tmppath = mkstemp(dir=self.swiftdir, suffix='.tmp.builder')
pickle.dump(builder.to_dict(), fdopen(fd, 'wb'), protocol=2)
backup, backup_md5 = make_backup(builder_file, self.backup_dir)
self.logger.notice('--> Backed up %s to %s (%s)' %
(builder_file, backup, backup_md5))
chmod(tmppath, 0644)
rename(tmppath, builder_file)
except Exception as err:
raise Exception('Error writing builder: %s' % err)
finally:
if fd:
try:
close(fd)
except OSError:
pass
if tmppath:
try:
unlink(tmppath)
except OSError:
pass
return get_md5sum(builder_file)
def __init__(self, conf):
self.current_md5 = {}
self.swiftdir = conf.get('swiftdir', '/etc/swift')
self.rings = {
'account':
conf.get('account_ring', pathjoin(self.swiftdir,
'account.ring.gz')),
'container':
conf.get('container_ring',
pathjoin(self.swiftdir, 'container.ring.gz')),
'object':
conf.get('object_ring', pathjoin(self.swiftdir, 'object.ring.gz'))
}
self.start_delay = int(conf.get('start_delay_range', '120'))
self.check_interval = int(conf.get('check_interval', '30'))
self.ring_master = conf.get('ring_master', 'http://127.0.0.1:8090/')
self.ring_master_timeout = int(conf.get('ring_master_timeout', '30'))
self.debug = conf.get('debug', 'n') in TRUE_VALUES
if self.debug:
conf['log_level'] = 'DEBUG'
self.logger = get_logger(conf, 'ringminiond', self.debug)
if not os.access(self.swiftdir, os.W_OK):
self.logger.error('swift_dir is not writable. exiting!')
sys.exit(1)
for ring in self.rings:
if exists(self.rings[ring]):
self.current_md5[self.rings[ring]] = \
get_md5sum(self.rings[ring])
else:
self.current_md5[self.rings[ring]] = ''
def _validate_file(self, filename):
"""Validate md5 of file"""
if self._changed(filename):
self.logger.debug("updating md5")
with lock_parent_directory(self.swiftdir, self.lock_timeout):
self.last_tstamp[filename] = stat(filename).st_mtime
self.current_md5[filename] = get_md5sum(filename)
def test_fetch_ring(self, fmoveinplace, fvalidatering, fwritering):
self._setup_obj_ring()
#test non 200
obj_ring = os.path.join(self.testdir, 'object.ring.gz')
obj_ring_md5 = get_md5sum(obj_ring)
minion = RingMinion(conf={'swiftdir': self.testdir})
minion.logger = MagicMock()
self.urlopen_mock.return_value = MockResponse(code=203)
result = minion.fetch_ring('object')
self.assertEquals(self.urlopen_mock.call_count, 1)
self.assertFalse(result)
urllib2.urlopen.assert_called_once
minion.logger.warning.assert_called_once_with('Received non 200 status code')
minion.logger.warning.reset_mock()
self.urlopen_mock.reset_mock()
#test 304
self.urlopen_mock.side_effect = urllib2.HTTPError('http://a.com', 304, 'Nope', {}, None)
minion.logger.debug.reset_mock()
minion.logger.warning.reset_mock()
minion.logger.exception.reset_mock()
result = minion.fetch_ring('object')
self.assertEquals(result, None)
minion.logger.debug.assert_called_with('Ring-master reports ring unchanged.')
minion.logger.debug.reset_mock()
#test HTTPError non 304
self.urlopen_mock.side_effect = urllib2.HTTPError('http://a.com', 401, 'GTFO', {}, None)
minion.logger.debug.reset_mock()
minion.logger.warning.reset_mock()
minion.logger.exception.reset_mock()
result = minion.fetch_ring('object')
self.assertFalse(result)
minion.logger.exception.assert_called_with('Error communicating with ring-master')
minion.logger.exception.reset_mock()
#test urllib2.URLError
self.urlopen_mock.side_effect = urllib2.URLError('oops')
minion.logger.debug.reset_mock()
minion.logger.warning.reset_mock()
minion.logger.exception.reset_mock()
result = minion.fetch_ring('object')
self.assertFalse(result)
minion.logger.exception.assert_called_with('Error communicating with ring-master')
minion.logger.exception.reset_mock()
#test general exception or timeout
self.urlopen_mock.side_effect = Exception('oopsie')
minion.logger.debug.reset_mock()
minion.logger.warning.reset_mock()
minion.logger.exception.reset_mock()
result = minion.fetch_ring('object')
self.assertFalse(result)
minion.logger.exception.assert_called_with('Error retrieving or checking on ring')
minion.logger.exception.reset_mock()
def __init__(self, conf):
self.ring_files = ["account.ring.gz", "container.ring.gz", "object.ring.gz"]
self.swiftdir = conf.get("swiftdir", "/etc/swift")
self.lock_timeout = float(conf.get("locktimeout", "5"))
self.wsgi_port = int(conf.get("serve_ring_port", "8090"))
self.wsgi_address = conf.get("serve_ring_address", "")
log_path = conf.get("log_path", "/var/log/ring-master/wsgi.log")
self.logger = get_file_logger("ring-master-wsgi", log_path)
self.last_tstamp = {}
self.current_md5 = {}
for rfile in self.ring_files:
target_file = pathjoin(self.swiftdir, rfile)
if exists(target_file):
self.last_tstamp[target_file] = stat(target_file).st_mtime
self.current_md5[target_file] = get_md5sum(target_file)
else:
self.last_tstamp[target_file] = None
self.request_logger = FileLikeLogger(self.logger)
def __init__(self, conf):
self.ring_files = ['account.ring.gz', 'container.ring.gz',
'object.ring.gz']
self.swiftdir = conf.get('swiftdir', '/etc/swift')
self.wsgi_port = int(conf.get('serve_ring_port', '8090'))
self.wsgi_address = conf.get('serve_ring_address', '')
log_path = conf.get('log_path', '/var/log/ring-master/wsgi.log')
self.logger = get_file_logger('ring-master-wsgi', log_path)
self.last_tstamp = {}
self.current_md5 = {}
for rfile in self.ring_files:
target_file = pathjoin(self.swiftdir, rfile)
if exists(target_file):
self.last_tstamp[target_file] = stat(target_file).st_mtime
self.current_md5[target_file] = get_md5sum(target_file)
else:
self.last_tstamp[target_file] = None
self.request_logger = FileLikeLogger(self.logger)
def test_handle_request(self):
self._setup_builder_rings()
start_response = MagicMock()
rma = RingMasterApp({'swiftdir': self.testdir, 'log_path': self.test_log_path})
# test bad path
req = Request.blank('/invalidrandomness',
environ={'REQUEST_METHOD': 'GET'})
resp = rma.handle_request(req.environ, start_response)
start_response.assert_called_with(
'404 Not Found', [('Content-Type', 'text/plain')])
self.assertEquals(resp, ['Not Found\r\n'])
# test legit path
req = Request.blank('/ring/account.ring.gz',
environ={'REQUEST_METHOD': 'HEAD'})
resp = rma.handle_request(req.environ, start_response)
account_md5 = get_md5sum(os.path.join(self.testdir, 'account.ring.gz'))
start_response.assert_called_with('200 OK', [('Content-Type', 'application/octet-stream'), ('Etag', account_md5)])
self.assertEquals(resp, [])
def __init__(self, conf):
self.ring_files = [
'account.ring.gz', 'container.ring.gz', 'object.ring.gz'
]
self.swiftdir = conf.get('swiftdir', '/etc/swift')
self.lock_timeout = float(conf.get('locktimeout', '5'))
self.wsgi_port = int(conf.get('serve_ring_port', '8090'))
self.wsgi_address = conf.get('serve_ring_address', '')
log_path = conf.get('log_path', '/var/log/ring-master/wsgi.log')
self.logger = get_file_logger('ring-master-wsgi', log_path)
self.last_tstamp = {}
self.current_md5 = {}
for rfile in self.ring_files:
target_file = pathjoin(self.swiftdir, rfile)
if exists(target_file):
self.last_tstamp[target_file] = stat(target_file).st_mtime
self.current_md5[target_file] = get_md5sum(target_file)
else:
self.last_tstamp[target_file] = None
self.request_logger = FileLikeLogger(self.logger)
def test_handle_request(self):
self._setup_builder_rings()
start_response = MagicMock()
rma = RingMasterApp({
'swiftdir': self.testdir,
'log_path': self.test_log_path
})
# test bad path
req = Request.blank('/invalidrandomness',
environ={'REQUEST_METHOD': 'GET'})
resp = rma.handle_request(req.environ, start_response)
start_response.assert_called_with('404 Not Found',
[('Content-Type', 'text/plain')])
self.assertEquals(resp, ['Not Found\r\n'])
# test legit path
req = Request.blank('/ring/account.ring.gz',
environ={'REQUEST_METHOD': 'HEAD'})
resp = rma.handle_request(req.environ, start_response)
account_md5 = get_md5sum(os.path.join(self.testdir, 'account.ring.gz'))
start_response.assert_called_with(
'200 OK', [('Content-Type', 'application/octet-stream'),
('Etag', account_md5)])
self.assertEquals(resp, [])
def _validate_file(self, filename):
"""Validate md5 of file"""
if self._changed(filename):
self.logger.debug("updating md5")
self.last_tstamp[filename] = stat(filename).st_mtime
self.current_md5[filename] = get_md5sum(filename)
def test_handle_ring(self):
self._setup_builder_rings()
start_response = MagicMock()
rma = RingMasterApp({'swiftdir': self.testdir, 'log_path': self.test_log_path})
# test bad path
req = Request.blank('/ring/not_a_valid_ring.gz',
environ={'REQUEST_METHOD': 'GET'})
resp = rma.handle_ring(req.environ, start_response)
start_response.assert_called_with(
'404 Not Found', [('Content-Type', 'text/plain')])
self.assertEquals(resp, ['Not Found\r\n'])
# test bad method
start_response.reset_mock()
req = Request.blank('/ring/account.ring.gz',
environ={'REQUEST_METHOD': 'DELETE'})
resp = rma.handle_ring(req.environ, start_response)
start_response.assert_called_with(
'501 Not Implemented', [('Content-Type', 'text/plain')])
self.assertEquals(resp, ['Not Implemented\r\n'])
# test HEAD
start_response.reset_mock()
req = Request.blank('/ring/account.ring.gz',
environ={'REQUEST_METHOD': 'HEAD'})
resp = rma.handle_ring(req.environ, start_response)
account_md5 = get_md5sum(os.path.join(self.testdir, 'account.ring.gz'))
start_response.assert_called_with('200 OK', [('Content-Type', 'application/octet-stream'), ('Etag', account_md5)])
self.assertEquals(resp, [])
# test GET w/ current If-None-Match
start_response.reset_mock()
account_md5 = get_md5sum(os.path.join(self.testdir, 'account.ring.gz'))
req = Request.blank('/ring/account.ring.gz',
environ={'REQUEST_METHOD': 'GET',
'HTTP_IF_NONE_MATCH': account_md5})
resp = rma.handle_ring(req.environ, start_response)
start_response.assert_called_with('304 Not Modified', [(
'Content-Type', 'application/octet-stream')])
self.assertEquals(resp, ['Not Modified\r\n'])
# test GET w/ outdated If-None-Match
start_response.reset_mock()
account_md5 = get_md5sum(os.path.join(self.testdir, 'account.ring.gz'))
req = Request.blank('/ring/account.ring.gz',
environ={'REQUEST_METHOD': 'GET',
'HTTP_IF_NONE_MATCH': 'ihazaring'})
resp = rma.handle_ring(req.environ, start_response)
start_response.assert_called_with('200 OK', [('Content-Type', 'application/octet-stream'), ('Etag', account_md5)])
testfile1 = os.path.join(self.testdir, 'gettest1.file')
with open(testfile1, 'w') as f:
for i in resp:
f.write(i)
self.assertTrue(account_md5, get_md5sum(testfile1))
# test GET without If-None-Match
start_response.reset_mock()
account_md5 = get_md5sum(os.path.join(self.testdir, 'account.ring.gz'))
req = Request.blank('/ring/account.ring.gz',
environ={'REQUEST_METHOD': 'GET'})
resp = rma.handle_ring(req.environ, start_response)
start_response.assert_called_with('200 OK', [('Content-Type', 'application/octet-stream'), ('Etag', account_md5)])
testfile2 = os.path.join(self.testdir, 'gettest2.file')
with open(testfile2, 'w') as f:
for i in resp:
f.write(i)
self.assertTrue(account_md5, get_md5sum(testfile2))
def test_handle_ring(self):
self._setup_builder_rings()
start_response = MagicMock()
rma = RingMasterApp({
'swiftdir': self.testdir,
'log_path': self.test_log_path
})
# test bad path
req = Request.blank('/ring/not_a_valid_ring.gz',
environ={'REQUEST_METHOD': 'GET'})
resp = rma.handle_ring(req.environ, start_response)
start_response.assert_called_with('404 Not Found',
[('Content-Type', 'text/plain')])
self.assertEquals(resp, ['Not Found\r\n'])
# test bad method
start_response.reset_mock()
req = Request.blank('/ring/account.ring.gz',
environ={'REQUEST_METHOD': 'DELETE'})
resp = rma.handle_ring(req.environ, start_response)
start_response.assert_called_with('501 Not Implemented',
[('Content-Type', 'text/plain')])
self.assertEquals(resp, ['Not Implemented\r\n'])
# test HEAD
start_response.reset_mock()
req = Request.blank('/ring/account.ring.gz',
environ={'REQUEST_METHOD': 'HEAD'})
resp = rma.handle_ring(req.environ, start_response)
account_md5 = get_md5sum(os.path.join(self.testdir, 'account.ring.gz'))
start_response.assert_called_with(
'200 OK', [('Content-Type', 'application/octet-stream'),
('Etag', account_md5)])
self.assertEquals(resp, [])
# test GET w/ current If-None-Match
start_response.reset_mock()
account_md5 = get_md5sum(os.path.join(self.testdir, 'account.ring.gz'))
req = Request.blank('/ring/account.ring.gz',
environ={
'REQUEST_METHOD': 'GET',
'HTTP_IF_NONE_MATCH': account_md5
})
resp = rma.handle_ring(req.environ, start_response)
start_response.assert_called_with(
'304 Not Modified', [('Content-Type', 'application/octet-stream')])
self.assertEquals(resp, ['Not Modified\r\n'])
# test GET w/ outdated If-None-Match
start_response.reset_mock()
account_md5 = get_md5sum(os.path.join(self.testdir, 'account.ring.gz'))
req = Request.blank('/ring/account.ring.gz',
environ={
'REQUEST_METHOD': 'GET',
'HTTP_IF_NONE_MATCH': 'ihazaring'
})
resp = rma.handle_ring(req.environ, start_response)
start_response.assert_called_with(
'200 OK', [('Content-Type', 'application/octet-stream'),
('Etag', account_md5)])
testfile1 = os.path.join(self.testdir, 'gettest1.file')
with open(testfile1, 'w') as f:
for i in resp:
f.write(i)
self.assertTrue(account_md5, get_md5sum(testfile1))
# test GET without If-None-Match
start_response.reset_mock()
account_md5 = get_md5sum(os.path.join(self.testdir, 'account.ring.gz'))
req = Request.blank('/ring/account.ring.gz',
environ={'REQUEST_METHOD': 'GET'})
resp = rma.handle_ring(req.environ, start_response)
start_response.assert_called_with(
'200 OK', [('Content-Type', 'application/octet-stream'),
('Etag', account_md5)])
testfile2 = os.path.join(self.testdir, 'gettest2.file')
with open(testfile2, 'w') as f:
for i in resp:
f.write(i)
self.assertTrue(account_md5, get_md5sum(testfile2))