def test_validate_ring(self):
     self._setup_obj_ring()
     #ok ring and ok md5
     obj_ring = os.path.join(self.testdir, 'object.ring.gz')
     obj_ring_md5 = get_md5sum(obj_ring)
     minion = RingMinion(conf={'swiftdir': self.testdir})
     minion._validate_ring(obj_ring, obj_ring_md5)
     # m5 miss match
     try:
         minion._validate_ring(obj_ring, 'badmd5')
     except Exception as err:
         self.assertEqual(err.message, "md5 missmatch")
     else:
         self.fail('Should have thrown md5 missmatch exception')
     # bad ring file
     bfile = os.path.join(self.testdir, 'test.ring.gz')
     with open(bfile, 'w') as f:
         f.write('whatisthis.')
     test_md5 = get_md5sum(bfile)
     try:
         minion._validate_ring(bfile, test_md5)
     except Exception as err:
         self.assertEqual(err.message, "Invalid ring")
     else:
         self.fail('Should have thrown Invalid ring exception')
 def _validate_file(self, filename):
     """Validate md5 of file"""
     if self._changed(filename):
         self.logger.debug("updating md5")
         with lock_parent_directory(self.swiftdir, self.lock_timeout):
             self.last_tstamp[filename] = stat(filename).st_mtime
             self.current_md5[filename] = get_md5sum(filename)
Пример #3
0
 def __init__(self, conf):
     self.current_md5 = {}
     self.swiftdir = conf.get('swiftdir', '/etc/swift')
     self.rings = {'account': conf.get('account_ring',
                                       pathjoin(self.swiftdir,
                                                'account.ring.gz')),
                   'container': conf.get('container_ring',
                                         pathjoin(self.swiftdir,
                                                  'container.ring.gz')),
                   'object': conf.get('object_ring',
                                      pathjoin(self.swiftdir,
                                               'object.ring.gz'))}
     self.start_delay = int(conf.get('start_delay_range', '120'))
     self.check_interval = int(conf.get('check_interval', '30'))
     self.ring_master = conf.get('ring_master', 'http://127.0.0.1:8090/')
     self.ring_master_timeout = int(conf.get('ring_master_timeout', '30'))
     self.debug = conf.get('debug', 'n') in TRUE_VALUES
     if self.debug:
         conf['log_level'] = 'DEBUG'
     self.logger = get_logger(conf, 'ringminiond', self.debug)
     if not os.access(self.swiftdir, os.W_OK):
         self.logger.error('swift_dir is not writable. exiting!')
         sys.exit(1)
     for ring in self.rings:
         if exists(self.rings[ring]):
             self.current_md5[self.rings[ring]] = \
                 get_md5sum(self.rings[ring])
         else:
             self.current_md5[self.rings[ring]] = ''
Пример #4
0
    def write_ring(self, btype, builder):
        """Write out new ring files

        :param btype: The builder type
        :param builder: The builder to dump
        :returns: new ring file md5
        """
        try:
            self.pause_if_asked()
            ring_file = self.ring_files[btype]
            fd, tmppath = mkstemp(dir=self.swiftdir, suffix='.tmp.ring.gz')
            builder.get_ring().save(tmppath)
            close(fd)
            if not is_valid_ring(tmppath):
                unlink(tmppath)
                raise Exception('Ring Validate Failed')
            backup, backup_md5 = make_backup(ring_file, self.backup_dir)
            self.logger.notice('--> Backed up %s to %s (%s)' %
                              (ring_file, backup, backup_md5))
            chmod(tmppath, 0644)
            rename(tmppath, ring_file)
        except Exception as err:
            raise Exception('Error writing builder: %s' % err)
        finally:
            if fd:
                try:
                    close(fd)
                except OSError:
                    pass
            if tmppath:
                try:
                    unlink(tmppath)
                except OSError:
                    pass
        return get_md5sum(ring_file)
Пример #5
0
    def write_builder(self, btype, builder):
        """Write out new builder file

        :param btype: The builder type
        :param builder: The builder to dump
        :returns: new ring file md5
        """
        self.pause_if_asked()
        builder_file = self.builder_files[btype]
        try:
            fd, tmppath = mkstemp(dir=self.swiftdir, suffix='.tmp.builder')
            pickle.dump(builder.to_dict(), fdopen(fd, 'wb'), protocol=2)
            backup, backup_md5 = make_backup(builder_file, self.backup_dir)
            self.logger.notice('--> Backed up %s to %s (%s)' %
                              (builder_file, backup, backup_md5))
            chmod(tmppath, 0644)
            rename(tmppath, builder_file)
        except Exception as err:
            raise Exception('Error writing builder: %s' % err)
        finally:
            if fd:
                try:
                    close(fd)
                except OSError:
                    pass
            if tmppath:
                try:
                    unlink(tmppath)
                except OSError:
                    pass
        return get_md5sum(builder_file)
Пример #6
0
 def __init__(self, conf):
     self.current_md5 = {}
     self.swiftdir = conf.get('swiftdir', '/etc/swift')
     self.rings = {
         'account':
         conf.get('account_ring', pathjoin(self.swiftdir,
                                           'account.ring.gz')),
         'container':
         conf.get('container_ring',
                  pathjoin(self.swiftdir, 'container.ring.gz')),
         'object':
         conf.get('object_ring', pathjoin(self.swiftdir, 'object.ring.gz'))
     }
     self.start_delay = int(conf.get('start_delay_range', '120'))
     self.check_interval = int(conf.get('check_interval', '30'))
     self.ring_master = conf.get('ring_master', 'http://127.0.0.1:8090/')
     self.ring_master_timeout = int(conf.get('ring_master_timeout', '30'))
     self.debug = conf.get('debug', 'n') in TRUE_VALUES
     if self.debug:
         conf['log_level'] = 'DEBUG'
     self.logger = get_logger(conf, 'ringminiond', self.debug)
     if not os.access(self.swiftdir, os.W_OK):
         self.logger.error('swift_dir is not writable. exiting!')
         sys.exit(1)
     for ring in self.rings:
         if exists(self.rings[ring]):
             self.current_md5[self.rings[ring]] = \
                 get_md5sum(self.rings[ring])
         else:
             self.current_md5[self.rings[ring]] = ''
Пример #7
0
 def _validate_file(self, filename):
     """Validate md5 of file"""
     if self._changed(filename):
         self.logger.debug("updating md5")
         with lock_parent_directory(self.swiftdir, self.lock_timeout):
             self.last_tstamp[filename] = stat(filename).st_mtime
             self.current_md5[filename] = get_md5sum(filename)
 def test_fetch_ring(self, fmoveinplace, fvalidatering, fwritering):
     self._setup_obj_ring()
     #test non 200
     obj_ring = os.path.join(self.testdir, 'object.ring.gz')
     obj_ring_md5 = get_md5sum(obj_ring)
     minion = RingMinion(conf={'swiftdir': self.testdir})
     minion.logger = MagicMock()
     self.urlopen_mock.return_value = MockResponse(code=203)
     result = minion.fetch_ring('object')
     self.assertEquals(self.urlopen_mock.call_count, 1)
     self.assertFalse(result)
     urllib2.urlopen.assert_called_once
     minion.logger.warning.assert_called_once_with('Received non 200 status code')
     minion.logger.warning.reset_mock()
     self.urlopen_mock.reset_mock()
     #test 304
     self.urlopen_mock.side_effect = urllib2.HTTPError('http://a.com', 304, 'Nope', {}, None)
     minion.logger.debug.reset_mock()
     minion.logger.warning.reset_mock()
     minion.logger.exception.reset_mock()
     result = minion.fetch_ring('object')
     self.assertEquals(result, None)
     minion.logger.debug.assert_called_with('Ring-master reports ring unchanged.')
     minion.logger.debug.reset_mock()
     #test HTTPError non 304
     self.urlopen_mock.side_effect = urllib2.HTTPError('http://a.com', 401, 'GTFO', {}, None)
     minion.logger.debug.reset_mock()
     minion.logger.warning.reset_mock()
     minion.logger.exception.reset_mock()
     result = minion.fetch_ring('object')
     self.assertFalse(result)
     minion.logger.exception.assert_called_with('Error communicating with ring-master')
     minion.logger.exception.reset_mock()
     #test urllib2.URLError
     self.urlopen_mock.side_effect = urllib2.URLError('oops')
     minion.logger.debug.reset_mock()
     minion.logger.warning.reset_mock()
     minion.logger.exception.reset_mock()
     result = minion.fetch_ring('object')
     self.assertFalse(result)
     minion.logger.exception.assert_called_with('Error communicating with ring-master')
     minion.logger.exception.reset_mock()
     #test general exception or timeout
     self.urlopen_mock.side_effect = Exception('oopsie')
     minion.logger.debug.reset_mock()
     minion.logger.warning.reset_mock()
     minion.logger.exception.reset_mock()
     result = minion.fetch_ring('object')
     self.assertFalse(result)
     minion.logger.exception.assert_called_with('Error retrieving or checking on ring')
     minion.logger.exception.reset_mock()
 def __init__(self, conf):
     self.ring_files = ["account.ring.gz", "container.ring.gz", "object.ring.gz"]
     self.swiftdir = conf.get("swiftdir", "/etc/swift")
     self.lock_timeout = float(conf.get("locktimeout", "5"))
     self.wsgi_port = int(conf.get("serve_ring_port", "8090"))
     self.wsgi_address = conf.get("serve_ring_address", "")
     log_path = conf.get("log_path", "/var/log/ring-master/wsgi.log")
     self.logger = get_file_logger("ring-master-wsgi", log_path)
     self.last_tstamp = {}
     self.current_md5 = {}
     for rfile in self.ring_files:
         target_file = pathjoin(self.swiftdir, rfile)
         if exists(target_file):
             self.last_tstamp[target_file] = stat(target_file).st_mtime
             self.current_md5[target_file] = get_md5sum(target_file)
         else:
             self.last_tstamp[target_file] = None
     self.request_logger = FileLikeLogger(self.logger)
Пример #10
0
 def __init__(self, conf):
     self.ring_files = ['account.ring.gz', 'container.ring.gz',
                        'object.ring.gz']
     self.swiftdir = conf.get('swiftdir', '/etc/swift')
     self.wsgi_port = int(conf.get('serve_ring_port', '8090'))
     self.wsgi_address = conf.get('serve_ring_address', '')
     log_path = conf.get('log_path', '/var/log/ring-master/wsgi.log')
     self.logger = get_file_logger('ring-master-wsgi', log_path)
     self.last_tstamp = {}
     self.current_md5 = {}
     for rfile in self.ring_files:
         target_file = pathjoin(self.swiftdir, rfile)
         if exists(target_file):
             self.last_tstamp[target_file] = stat(target_file).st_mtime
             self.current_md5[target_file] = get_md5sum(target_file)
         else:
             self.last_tstamp[target_file] = None
     self.request_logger = FileLikeLogger(self.logger)
Пример #11
0
 def test_handle_request(self):
     self._setup_builder_rings()
     start_response = MagicMock()
     rma = RingMasterApp({'swiftdir': self.testdir, 'log_path': self.test_log_path})
     # test bad path
     req = Request.blank('/invalidrandomness',
                         environ={'REQUEST_METHOD': 'GET'})
     resp = rma.handle_request(req.environ, start_response)
     start_response.assert_called_with(
         '404 Not Found', [('Content-Type', 'text/plain')])
     self.assertEquals(resp, ['Not Found\r\n'])
     # test legit path
     req = Request.blank('/ring/account.ring.gz',
                         environ={'REQUEST_METHOD': 'HEAD'})
     resp = rma.handle_request(req.environ, start_response)
     account_md5 = get_md5sum(os.path.join(self.testdir, 'account.ring.gz'))
     start_response.assert_called_with('200 OK', [('Content-Type', 'application/octet-stream'), ('Etag', account_md5)])
     self.assertEquals(resp, [])
Пример #12
0
 def __init__(self, conf):
     self.ring_files = [
         'account.ring.gz', 'container.ring.gz', 'object.ring.gz'
     ]
     self.swiftdir = conf.get('swiftdir', '/etc/swift')
     self.lock_timeout = float(conf.get('locktimeout', '5'))
     self.wsgi_port = int(conf.get('serve_ring_port', '8090'))
     self.wsgi_address = conf.get('serve_ring_address', '')
     log_path = conf.get('log_path', '/var/log/ring-master/wsgi.log')
     self.logger = get_file_logger('ring-master-wsgi', log_path)
     self.last_tstamp = {}
     self.current_md5 = {}
     for rfile in self.ring_files:
         target_file = pathjoin(self.swiftdir, rfile)
         if exists(target_file):
             self.last_tstamp[target_file] = stat(target_file).st_mtime
             self.current_md5[target_file] = get_md5sum(target_file)
         else:
             self.last_tstamp[target_file] = None
     self.request_logger = FileLikeLogger(self.logger)
Пример #13
0
 def test_handle_request(self):
     self._setup_builder_rings()
     start_response = MagicMock()
     rma = RingMasterApp({
         'swiftdir': self.testdir,
         'log_path': self.test_log_path
     })
     # test bad path
     req = Request.blank('/invalidrandomness',
                         environ={'REQUEST_METHOD': 'GET'})
     resp = rma.handle_request(req.environ, start_response)
     start_response.assert_called_with('404 Not Found',
                                       [('Content-Type', 'text/plain')])
     self.assertEquals(resp, ['Not Found\r\n'])
     # test legit path
     req = Request.blank('/ring/account.ring.gz',
                         environ={'REQUEST_METHOD': 'HEAD'})
     resp = rma.handle_request(req.environ, start_response)
     account_md5 = get_md5sum(os.path.join(self.testdir, 'account.ring.gz'))
     start_response.assert_called_with(
         '200 OK', [('Content-Type', 'application/octet-stream'),
                    ('Etag', account_md5)])
     self.assertEquals(resp, [])
Пример #14
0
 def _validate_file(self, filename):
     """Validate md5 of file"""
     if self._changed(filename):
         self.logger.debug("updating md5")
         self.last_tstamp[filename] = stat(filename).st_mtime
         self.current_md5[filename] = get_md5sum(filename)
Пример #15
0
    def test_handle_ring(self):
        self._setup_builder_rings()
        start_response = MagicMock()
        rma = RingMasterApp({'swiftdir': self.testdir, 'log_path': self.test_log_path})

        # test bad path
        req = Request.blank('/ring/not_a_valid_ring.gz',
                            environ={'REQUEST_METHOD': 'GET'})
        resp = rma.handle_ring(req.environ, start_response)
        start_response.assert_called_with(
            '404 Not Found', [('Content-Type', 'text/plain')])
        self.assertEquals(resp, ['Not Found\r\n'])

        # test bad method
        start_response.reset_mock()
        req = Request.blank('/ring/account.ring.gz',
                            environ={'REQUEST_METHOD': 'DELETE'})
        resp = rma.handle_ring(req.environ, start_response)
        start_response.assert_called_with(
            '501 Not Implemented', [('Content-Type', 'text/plain')])
        self.assertEquals(resp, ['Not Implemented\r\n'])

        # test HEAD
        start_response.reset_mock()
        req = Request.blank('/ring/account.ring.gz',
                            environ={'REQUEST_METHOD': 'HEAD'})
        resp = rma.handle_ring(req.environ, start_response)
        account_md5 = get_md5sum(os.path.join(self.testdir, 'account.ring.gz'))
        start_response.assert_called_with('200 OK', [('Content-Type', 'application/octet-stream'), ('Etag', account_md5)])
        self.assertEquals(resp, [])

        # test GET w/ current If-None-Match
        start_response.reset_mock()
        account_md5 = get_md5sum(os.path.join(self.testdir, 'account.ring.gz'))
        req = Request.blank('/ring/account.ring.gz',
                            environ={'REQUEST_METHOD': 'GET',
                                     'HTTP_IF_NONE_MATCH': account_md5})
        resp = rma.handle_ring(req.environ, start_response)
        start_response.assert_called_with('304 Not Modified', [(
            'Content-Type', 'application/octet-stream')])
        self.assertEquals(resp, ['Not Modified\r\n'])

        # test GET w/ outdated If-None-Match
        start_response.reset_mock()
        account_md5 = get_md5sum(os.path.join(self.testdir, 'account.ring.gz'))
        req = Request.blank('/ring/account.ring.gz',
                            environ={'REQUEST_METHOD': 'GET',
                                     'HTTP_IF_NONE_MATCH': 'ihazaring'})
        resp = rma.handle_ring(req.environ, start_response)
        start_response.assert_called_with('200 OK', [('Content-Type', 'application/octet-stream'), ('Etag', account_md5)])
        testfile1 = os.path.join(self.testdir, 'gettest1.file')
        with open(testfile1, 'w') as f:
            for i in resp:
                f.write(i)
        self.assertTrue(account_md5, get_md5sum(testfile1))

        # test GET without If-None-Match
        start_response.reset_mock()
        account_md5 = get_md5sum(os.path.join(self.testdir, 'account.ring.gz'))
        req = Request.blank('/ring/account.ring.gz',
                            environ={'REQUEST_METHOD': 'GET'})
        resp = rma.handle_ring(req.environ, start_response)
        start_response.assert_called_with('200 OK', [('Content-Type', 'application/octet-stream'), ('Etag', account_md5)])
        testfile2 = os.path.join(self.testdir, 'gettest2.file')
        with open(testfile2, 'w') as f:
            for i in resp:
                f.write(i)
        self.assertTrue(account_md5, get_md5sum(testfile2))
Пример #16
0
    def test_handle_ring(self):
        self._setup_builder_rings()
        start_response = MagicMock()
        rma = RingMasterApp({
            'swiftdir': self.testdir,
            'log_path': self.test_log_path
        })

        # test bad path
        req = Request.blank('/ring/not_a_valid_ring.gz',
                            environ={'REQUEST_METHOD': 'GET'})
        resp = rma.handle_ring(req.environ, start_response)
        start_response.assert_called_with('404 Not Found',
                                          [('Content-Type', 'text/plain')])
        self.assertEquals(resp, ['Not Found\r\n'])

        # test bad method
        start_response.reset_mock()
        req = Request.blank('/ring/account.ring.gz',
                            environ={'REQUEST_METHOD': 'DELETE'})
        resp = rma.handle_ring(req.environ, start_response)
        start_response.assert_called_with('501 Not Implemented',
                                          [('Content-Type', 'text/plain')])
        self.assertEquals(resp, ['Not Implemented\r\n'])

        # test HEAD
        start_response.reset_mock()
        req = Request.blank('/ring/account.ring.gz',
                            environ={'REQUEST_METHOD': 'HEAD'})
        resp = rma.handle_ring(req.environ, start_response)
        account_md5 = get_md5sum(os.path.join(self.testdir, 'account.ring.gz'))
        start_response.assert_called_with(
            '200 OK', [('Content-Type', 'application/octet-stream'),
                       ('Etag', account_md5)])
        self.assertEquals(resp, [])

        # test GET w/ current If-None-Match
        start_response.reset_mock()
        account_md5 = get_md5sum(os.path.join(self.testdir, 'account.ring.gz'))
        req = Request.blank('/ring/account.ring.gz',
                            environ={
                                'REQUEST_METHOD': 'GET',
                                'HTTP_IF_NONE_MATCH': account_md5
                            })
        resp = rma.handle_ring(req.environ, start_response)
        start_response.assert_called_with(
            '304 Not Modified', [('Content-Type', 'application/octet-stream')])
        self.assertEquals(resp, ['Not Modified\r\n'])

        # test GET w/ outdated If-None-Match
        start_response.reset_mock()
        account_md5 = get_md5sum(os.path.join(self.testdir, 'account.ring.gz'))
        req = Request.blank('/ring/account.ring.gz',
                            environ={
                                'REQUEST_METHOD': 'GET',
                                'HTTP_IF_NONE_MATCH': 'ihazaring'
                            })
        resp = rma.handle_ring(req.environ, start_response)
        start_response.assert_called_with(
            '200 OK', [('Content-Type', 'application/octet-stream'),
                       ('Etag', account_md5)])
        testfile1 = os.path.join(self.testdir, 'gettest1.file')
        with open(testfile1, 'w') as f:
            for i in resp:
                f.write(i)
        self.assertTrue(account_md5, get_md5sum(testfile1))

        # test GET without If-None-Match
        start_response.reset_mock()
        account_md5 = get_md5sum(os.path.join(self.testdir, 'account.ring.gz'))
        req = Request.blank('/ring/account.ring.gz',
                            environ={'REQUEST_METHOD': 'GET'})
        resp = rma.handle_ring(req.environ, start_response)
        start_response.assert_called_with(
            '200 OK', [('Content-Type', 'application/octet-stream'),
                       ('Etag', account_md5)])
        testfile2 = os.path.join(self.testdir, 'gettest2.file')
        with open(testfile2, 'w') as f:
            for i in resp:
                f.write(i)
        self.assertTrue(account_md5, get_md5sum(testfile2))