def setUp(self):
self.conf = SyncObjConf(appendEntriesUseBatch=False, appendEntriesPeriod=0.001,
raftMinTimeout=0.004, raftMaxTimeout=0.005, autoTickPeriod=0.001)
callback = Mock()
callback.replicated = False
self.so = KVStoreTTL('127.0.0.1:1234', [], self.conf, on_set=callback, on_delete=callback)
self.so.set_retry_timeout(10)
def setUp(self):
callback = Mock()
callback.replicated = False
self.so = KVStoreTTL(None,
callback,
callback,
self_addr='127.0.0.1:1234')
self.so.startAutoTick()
self.so.set_retry_timeout(10)
def cleanup_service_tree(self):
from patroni.dcs.raft import KVStoreTTL
if self._raft:
self._raft.destroy()
self.stop()
os.makedirs(self._work_directory)
self.start()
ready_event = threading.Event()
self._raft = KVStoreTTL(ready_event.set, None, None, partner_addrs=[self.CONTROLLER_ADDR], password=self.PASSWORD)
self._raft.startAutoTick()
ready_event.wait()
class RaftController(AbstractDcsController):
CONTROLLER_ADDR = 'localhost:1234'
PASSWORD = '******'
def __init__(self, context):
super(RaftController, self).__init__(context)
os.environ.update(PATRONI_RAFT_PARTNER_ADDRS="'" +
self.CONTROLLER_ADDR + "'",
PATRONI_RAFT_PASSWORD=self.PASSWORD,
RAFT_PORT='1234')
self._raft = None
def _start(self):
env = os.environ.copy()
del env['PATRONI_RAFT_PARTNER_ADDRS']
env['PATRONI_RAFT_SELF_ADDR'] = self.CONTROLLER_ADDR
env['PATRONI_RAFT_DATA_DIR'] = self._work_directory
return subprocess.Popen([
sys.executable, '-m', 'coverage', 'run', '--source=patroni', '-p',
'patroni_raft_controller.py'
],
stdout=self._log,
stderr=subprocess.STDOUT,
env=env)
def query(self, key, scope='batman'):
ret = self._raft.get(self.path(key, scope))
return ret and ret['value']
def set(self, key, value):
self._raft.set(self.path(key), value)
def cleanup_service_tree(self):
from patroni.dcs.raft import KVStoreTTL
if self._raft:
self._raft.destroy()
self.stop()
os.makedirs(self._work_directory)
self.start()
ready_event = threading.Event()
self._raft = KVStoreTTL(ready_event.set,
None,
None,
partner_addrs=[self.CONTROLLER_ADDR],
password=self.PASSWORD)
self._raft.startAutoTick()
ready_event.wait()
def __init__(self, config):
super(RaftController, self).__init__(config)
raft_config = self.config.get('raft')
self_addr = raft_config['self_addr']
template = os.path.join(raft_config.get('data_dir', ''), self_addr)
self._syncobj_config = SyncObjConf(autoTick=False,
appendEntriesUseBatch=False,
dynamicMembershipChange=True,
journalFile=template + '.journal',
fullDumpFile=template + '.dump')
self._raft = KVStoreTTL(self_addr,
raft_config.get('partner_addrs',
[]), self._syncobj_config)
def test_on_ready_override(self):
self.assertTrue(self.so.set('foo', 'bar'))
self.so.destroy()
self.so = None
so = KVStoreTTL(Mock(), None, None, self_addr='127.0.0.1:1234',
partner_addrs=['127.0.0.1:1235'], patronictl=True)
so.doTick(0)
so.destroy()
def test_on_ready_override(self):
self.assertTrue(self.so.set('foo', 'bar'))
self.destroy(self.so)
self.so = None
self.conf.onReady = Mock()
self.conf.autoTick = False
so = KVStoreTTL('127.0.0.1:1234', ['127.0.0.1:1235'], self.conf)
so.doTick(0)
so.destroy()
class RaftController(AbstractPatroniDaemon):
def __init__(self, config):
super(RaftController, self).__init__(config)
raft_config = self.config.get('raft')
self_addr = raft_config['self_addr']
template = os.path.join(raft_config.get('data_dir', ''), self_addr)
self._syncobj_config = SyncObjConf(autoTick=False,
appendEntriesUseBatch=False,
dynamicMembershipChange=True,
journalFile=template + '.journal',
fullDumpFile=template + '.dump')
self._raft = KVStoreTTL(self_addr,
raft_config.get('partner_addrs',
[]), self._syncobj_config)
def _run_cycle(self):
try:
self._raft.doTick(self._syncobj_config.autoTickPeriod)
except Exception:
logger.exception('doTick')
def _shutdown(self):
self._raft.destroy()
def cleanup_service_tree(self):
from patroni.dcs.raft import KVStoreTTL
from pysyncobj import SyncObjConf
if self._raft:
self._raft.destroy()
self._raft._SyncObj__thread.join()
self.stop()
os.makedirs(self._work_directory)
self.start()
ready_event = threading.Event()
conf = SyncObjConf(appendEntriesUseBatch=False, dynamicMembershipChange=True, onReady=ready_event.set)
self._raft = KVStoreTTL(None, [self.CONTROLLER_ADDR], conf)
ready_event.wait()
class TestKVStoreTTL(unittest.TestCase):
def setUp(self):
self.conf = SyncObjConf(appendEntriesUseBatch=False, appendEntriesPeriod=0.001,
raftMinTimeout=0.004, raftMaxTimeout=0.005, autoTickPeriod=0.001)
callback = Mock()
callback.replicated = False
self.so = KVStoreTTL('127.0.0.1:1234', [], self.conf, on_set=callback, on_delete=callback)
self.so.set_retry_timeout(10)
@staticmethod
def destroy(so):
so.destroy()
so._SyncObj__thread.join()
def tearDown(self):
if self.so:
self.destroy(self.so)
def test_set(self):
self.assertTrue(self.so.set('foo', 'bar', prevExist=False, ttl=30))
self.assertFalse(self.so.set('foo', 'bar', prevExist=False, ttl=30))
self.assertFalse(self.so.retry(self.so._set, 'foo', {'value': 'buz', 'created': 1, 'updated': 1}, prevValue=''))
self.assertTrue(self.so.retry(self.so._set, 'foo', {'value': 'buz', 'created': 1, 'updated': 1}))
def test_delete(self):
self.conf.autoTickPeriod = 0.1
self.so.set('foo', 'bar')
self.so.set('fooo', 'bar')
self.assertFalse(self.so.delete('foo', prevValue='buz'))
self.assertTrue(self.so.delete('foo', recursive=True))
self.assertFalse(self.so.retry(self.so._delete, 'foo', prevValue=''))
def test_expire(self):
self.so.set('foo', 'bar', ttl=0.001)
time.sleep(1)
self.assertIsNone(self.so.get('foo'))
self.assertEqual(self.so.get('foo', recursive=True), {})
@patch('time.sleep', Mock())
def test_retry(self):
return_values = [FAIL_REASON.QUEUE_FULL] * 2 + [FAIL_REASON.SUCCESS, FAIL_REASON.REQUEST_DENIED]
def test(callback):
callback(True, return_values.pop(0))
with patch('time.time', Mock(side_effect=[1, 100])):
self.assertFalse(self.so.retry(test))
self.assertTrue(self.so.retry(test))
self.assertFalse(self.so.retry(test))
def test_on_ready_override(self):
self.assertTrue(self.so.set('foo', 'bar'))
self.destroy(self.so)
self.so = None
self.conf.onReady = Mock()
self.conf.autoTick = False
so = KVStoreTTL('127.0.0.1:1234', ['127.0.0.1:1235'], self.conf)
so.doTick(0)
so.destroy()
class TestKVStoreTTL(unittest.TestCase):
@patch.object(SyncObjConf, 'fullDumpFile', PropertyMock(return_value=None), create=True)
@patch.object(SyncObjConf, 'journalFile', PropertyMock(return_value=None), create=True)
def setUp(self):
callback = Mock()
callback.replicated = False
self.so = KVStoreTTL(None, callback, callback, self_addr='127.0.0.1:1234')
self.so.startAutoTick()
self.so.set_retry_timeout(10)
def tearDown(self):
if self.so:
self.so.destroy()
def test_set(self):
self.assertTrue(self.so.set('foo', 'bar', prevExist=False, ttl=30))
self.assertFalse(self.so.set('foo', 'bar', prevExist=False, ttl=30))
self.assertFalse(self.so.retry(self.so._set, 'foo', {'value': 'buz', 'created': 1, 'updated': 1}, prevValue=''))
self.assertTrue(self.so.retry(self.so._set, 'foo', {'value': 'buz', 'created': 1, 'updated': 1}))
def test_delete(self):
self.so.autoTickPeriod = 0.2
self.so.set('foo', 'bar')
self.so.set('fooo', 'bar')
self.assertFalse(self.so.delete('foo', prevValue='buz'))
self.assertTrue(self.so.delete('foo', recursive=True))
self.assertFalse(self.so.retry(self.so._delete, 'foo', prevValue=''))
def test_expire(self):
self.so.set('foo', 'bar', ttl=0.001)
time.sleep(1)
self.assertIsNone(self.so.get('foo'))
self.assertEqual(self.so.get('foo', recursive=True), {})
@patch('time.sleep', Mock())
def test_retry(self):
return_values = [FAIL_REASON.QUEUE_FULL] * 2 + [FAIL_REASON.SUCCESS, FAIL_REASON.REQUEST_DENIED]
def test(callback):
callback(True, return_values.pop(0))
with patch('time.time', Mock(side_effect=[1, 100])):
self.assertFalse(self.so.retry(test))
self.assertTrue(self.so.retry(test))
self.assertFalse(self.so.retry(test))
def test_on_ready_override(self):
self.assertTrue(self.so.set('foo', 'bar'))
self.so.destroy()
self.so = None
so = KVStoreTTL(Mock(), None, None, self_addr='127.0.0.1:1234',
partner_addrs=['127.0.0.1:1235'], patronictl=True)
so.doTick(0)
so.destroy()