def test_auto_reconnect_node():
# If a connection with current node is lost then the client automatically
# connects to another.
proxied_clt = _fixture.proxied_clt()
proxied_clt.put('/test/foo', 'bar')
_assert_get_one('bar', proxied_clt.get('/test/foo'))
orig_endpoint = proxied_clt.current_endpoint
i = 0
while True:
val = 'bazz%d' % (i,)
endpoint = proxied_clt.current_endpoint
# When
_fixture.disable_endpoint(endpoint)
try:
# Then
proxied_clt.put('/test/foo', val)
assert_not_equal(endpoint, proxied_clt.current_endpoint)
_assert_get_one(val, proxied_clt.get('/test/foo'))
finally:
_fixture.enable_endpoint(endpoint)
# Eventually we should cycle through all endpoints and get back to the
# origin.
if endpoint == orig_endpoint:
break
def test_watch_from_the_past():
# A revision to start watching from can be specified.
proxied_clt = _fixture.proxied_clt()
watch_queue = queue.Queue()
proxied_clt.put('/test/foo2', 'bar1')
proxied_clt.put('/test/foo2', 'bar2')
put_rs = proxied_clt.put('/test/foo2', 'bar3')
proxied_clt.put('/test/foo2', 'bar4')
proxied_clt.put('/test/foo2', 'bar5')
proxied_clt.put('/test/foo3', 'bar6')
# When
w = proxied_clt.new_watcher('/test/foo2',
spin_pause=0.2,
start_revision=put_rs.header.revision,
event_handler=lambda e: watch_queue.put(e))
w.start()
try:
proxied_clt.delete('/test/foo2')
# Then
events = _collect_events(watch_queue, timeout=3)
eq_(4, len(events))
_assert_event(Event.PUT, '/test/foo2', 'bar3', events[0])
_assert_event(Event.PUT, '/test/foo2', 'bar4', events[1])
_assert_event(Event.PUT, '/test/foo2', 'bar5', events[2])
_assert_event(Event.DELETE, '/test/foo2', '', events[3])
finally:
w.stop(timeout=1)
def test_watch_key_prefix():
proxied_clt = _fixture.proxied_clt()
watch_queue = queue.Queue()
proxied_clt.put('/test/foo2', 'bar1')
w = proxied_clt.new_watcher('/test/foo2',
is_prefix=True,
spin_pause=0.2,
event_handler=lambda e: watch_queue.put(e))
w.start()
try:
sleep(0.5)
# When
proxied_clt.put('/test/foo1', 'bar6')
proxied_clt.put('/test/foo21', 'bar7')
proxied_clt.delete('/test/foo2')
proxied_clt.put('/test/foo212', 'bar8')
proxied_clt.delete('/test/foo1')
proxied_clt.put('/test/foo21', 'bar10')
proxied_clt.put('/test/foo3', 'bar11')
# Then
events = _collect_events(watch_queue, timeout=3)
eq_(4, len(events))
_assert_event(Event.PUT, '/test/foo21', 'bar7', events[0])
_assert_event(Event.DELETE, '/test/foo2', '', events[1])
_assert_event(Event.PUT, '/test/foo212', 'bar8', events[2])
_assert_event(Event.PUT, '/test/foo21', 'bar10', events[3])
finally:
w.stop(timeout=1)
def test_etcd_down_on_start():
# It is ok to create a keep_aliver when connection with Etcd is down. The
# key will be created as soon as connection is restored.
proxied_clt = _fixture.proxied_clt()
# Simulate Etcd down on start.
_fixture.disable_all_endpoints()
ttl = 2
keep_aliver = proxied_clt.new_keep_aliver('/test/keep-alive-3',
'bar',
ttl,
spin_pause=0.2)
keep_aliver.start()
try:
sleep(0.5)
eq_(None, _fixture.aux_clt().get_value('/test/keep-alive-3'))
# When: etcd gets back in service.
_fixture.enabled_all_endpoints()
# Then: the key is automatically created.
sleep(0.5)
eq_(b'bar', proxied_clt.get_value('/test/keep-alive-3'))
sleep(ttl - 0.5)
eq_(b'bar', proxied_clt.get_value('/test/keep-alive-3'))
finally:
eq_(True, keep_aliver.stop(timeout=3))
eq_(None, proxied_clt.get_value('/test/keep-alive-3'))
def test_get_does_not_exist():
proxied_clt = _fixture.proxied_clt()
# When
rs = proxied_clt.get('/test/foo')
# Then
_assert_get_one(None, rs)
def test_lease_revoke():
proxied_clt = _fixture.proxied_clt()
rs = proxied_clt.lease_grant(600)
proxied_clt.put('/test/foo', 'bar', rs.ID)
_assert_get_one('bar', proxied_clt.get('/test/foo'))
# When
proxied_clt.lease_revoke(rs.ID)
# Then
_assert_get_one(None, proxied_clt.get('/test/foo'))
def test_lease_expires():
proxied_clt = _fixture.proxied_clt()
rs = proxied_clt.lease_grant(1)
proxied_clt.put('/test/foo', 'bar', rs.ID)
_assert_get_one('bar', proxied_clt.get('/test/foo'))
# When
sleep(rs.TTL + 1)
# Then
_assert_get_one(None, proxied_clt.get('/test/foo'))
def test_get_prefix_limit():
proxied_clt = _fixture.proxied_clt()
proxied_clt.put('/test/foo1', 'bar5')
proxied_clt.put('/test/foo2', 'bar4')
proxied_clt.put('/test/foo21', 'bar3')
proxied_clt.put('/test/foo212', 'bar2')
proxied_clt.put('/test/foo3', 'bar1')
# When
rs = proxied_clt.get('/test/foo', is_prefix=True, limit=2)
# Then
eq_(5, rs.count)
eq_(2, len(rs.kvs))
def test_get_prefix():
proxied_clt = _fixture.proxied_clt()
proxied_clt.put('/test/foo1', 'bar1')
proxied_clt.put('/test/foo2', 'bar2')
proxied_clt.put('/test/foo21', 'bar3')
proxied_clt.put('/test/foo212', 'bar4')
proxied_clt.put('/test/foo3', 'bar5')
# When
rs = proxied_clt.get('/test/foo2', is_prefix=True)
# Then
eq_(3, rs.count)
eq_(3, len(rs.kvs))
eq_(b'bar2', rs.kvs[0].value)
eq_(b'bar3', rs.kvs[1].value)
eq_(b'bar4', rs.kvs[2].value)
def test_delete_range():
proxied_clt = _fixture.proxied_clt()
proxied_clt.put('/test/foo1', 'bar1')
proxied_clt.put('/test/foo2', 'bar2')
proxied_clt.put('/test/foo21', 'bar3')
proxied_clt.put('/test/foo212', 'bar4')
proxied_clt.put('/test/foo3', 'bar5')
# When
rs = proxied_clt.delete('/test/foo2', is_prefix=True)
# Then
eq_(3, rs.deleted)
_assert_get_one('bar1', proxied_clt.get('/test/foo1'))
_assert_get_one(None, proxied_clt.get('/test/foo2'))
_assert_get_one(None, proxied_clt.get('/test/foo21'))
_assert_get_one(None, proxied_clt.get('/test/foo212'))
_assert_get_one('bar5', proxied_clt.get('/test/foo3'))
def test_spin_pause_too_long():
# Spin pause is adjusted if needed to be less then effective TTL.
proxied_clt = _fixture.proxied_clt()
ttl = 3
keep_aliver = proxied_clt.new_keep_aliver('/test/keep-alive-1',
'bar',
ttl,
spin_pause=10)
keep_aliver.start()
try:
sleep(ttl - 0.5)
eq_(b'bar', proxied_clt.get_value('/test/keep-alive-1'))
finally:
eq_(True, keep_aliver.stop(timeout=3))
eq_(None, proxied_clt.get_value('/test/keep-alive-1'))
def test_get_revisions():
# Checks that create_revision and mod_revision are returned correctly.
proxied_clt = _fixture.proxied_clt()
# Given
proxied_clt.put('/test/foo0', 'tox')
create_rs = proxied_clt.put('/test/foo', 'bar')
proxied_clt.put('/test/foo', 'bazz')
proxied_clt.put('/test/foo2', 'fox')
update_rs = proxied_clt.put('/test/foo', 'blah')
proxied_clt.put('/test/foo3', 'socks')
# When
rs = proxied_clt.get('/test/foo')
# Then
eq_(create_rs.header.revision, rs.kvs[0].create_revision)
eq_(update_rs.header.revision, rs.kvs[0].mod_revision)
_assert_get_one('blah', rs)
def test_normal_operation():
# KeepAliver ensure that key exists while running and deletes it on stop.
proxied_clt = _fixture.proxied_clt()
ttl = 2
keep_aliver = proxied_clt.new_keep_aliver('/test/keep-alive-0',
'bar',
ttl,
spin_pause=0.2)
keep_aliver.start()
try:
for i in range(3):
sleep(ttl - 0.5)
eq_(b'bar', proxied_clt.get_value('/test/keep-alive-0'))
finally:
eq_(True, keep_aliver.stop(timeout=3))
eq_(None, proxied_clt.get_value('/test/keep-alive-0'))
def test_auto_reconnect_outage():
# Watch is restored even after a cluster wide outage.
proxied_clt = _fixture.proxied_clt()
watch_queue = queue.Queue()
put_rs = _fixture.aux_clt().put('/test/foo', 'bar0')
w = proxied_clt.new_watcher('/test/foo',
spin_pause=0.2,
start_revision=put_rs.header.revision + 1,
event_handler=lambda e: watch_queue.put(e))
w.start()
try:
_fixture.aux_clt().put('/test/foo', 'bar1')
_fixture.aux_clt().put('/test/foo', 'bar2')
sleep(0.5)
# When connection with Etcd is lost...
_fixture.disable_all_endpoints()
# New events stop coming.
_fixture.aux_clt().put('/test/foo', 'bar3')
_fixture.aux_clt().delete('/test/foo')
_fixture.aux_clt().put('/test/foo', 'bar4')
events = _collect_events(watch_queue, timeout=3)
eq_(2, len(events))
_assert_event(Event.PUT, '/test/foo', 'bar1', events[0])
_assert_event(Event.PUT, '/test/foo', 'bar2', events[1])
# But as soon as the cluster is back in service...
_fixture.enabled_all_endpoints()
# ...backed up events are received
events = _collect_events(watch_queue, timeout=5)
eq_(3, len(events))
_assert_event(Event.PUT, '/test/foo', 'bar3', events[0])
_assert_event(Event.DELETE, '/test/foo', '', events[1])
_assert_event(Event.PUT, '/test/foo', 'bar4', events[2])
finally:
w.stop(timeout=1)
def test_get_prefix_sort():
proxied_clt = _fixture.proxied_clt()
proxied_clt.put('/test/foo212', 'bar4')
proxied_clt.put('/test/foo2', 'bar2')
proxied_clt.put('/test/foo3', 'bar1')
proxied_clt.put('/test/foo1', 'bar5')
proxied_clt.put('/test/foo21', 'bar3')
for i, tc in enumerate([{
'order': SortOrder.ASCEND,
'target': SortTarget.KEY,
'out': [b'bar5', b'bar2', b'bar3', b'bar4', b'bar1'],
}, {
'order': SortOrder.ASCEND,
'target': SortTarget.VALUE,
'out': [b'bar1', b'bar2', b'bar3', b'bar4', b'bar5'],
}, {
'order': SortOrder.ASCEND,
'target': SortTarget.CREATE_REVISION,
'out': [b'bar4', b'bar2', b'bar1', b'bar5', b'bar3'],
}, {
'order': SortOrder.DESCEND,
'target': SortTarget.KEY,
'out': [b'bar1', b'bar4', b'bar3', b'bar2', b'bar5'],
}, {
'order': SortOrder.DESCEND,
'target': SortTarget.VALUE,
'out': [b'bar5', b'bar4', b'bar3', b'bar2', b'bar1'],
}, {
'order': SortOrder.DESCEND,
'target': SortTarget.CREATE_REVISION,
'out': [b'bar3', b'bar5', b'bar1', b'bar2', b'bar4'],
}]):
print('Test case #%d: %s/%s', i, tc['order'], tc['target'])
# When
rs = proxied_clt.get('/test/foo', is_prefix=True,
sort_order=tc['order'], sort_target=tc['target'])
# Then
eq_(tc['out'], [kv.value for kv in rs.kvs])
def test_auto_reconnect_node():
# Watch is restored even after a cluster wide outage.
proxied_clt = _fixture.proxied_clt()
watch_queue = queue.Queue()
put_rs = _fixture.aux_clt().put('/test/foo', 'bar0')
w = proxied_clt.new_watcher('/test/foo',
spin_pause=0.2,
start_revision=put_rs.header.revision + 1,
event_handler=lambda e: watch_queue.put(e))
w.start()
try:
_fixture.aux_clt().put('/test/foo', 'bar1')
events = _collect_events(watch_queue, timeout=3)
eq_(1, len(events))
orig_endpoint = proxied_clt.current_endpoint
_fixture.aux_clt().put('/test/foo', 'bar2')
_fixture.aux_clt().put('/test/foo', 'bar3')
_fixture.aux_clt().put('/test/foo', 'bar4')
events = _collect_events(watch_queue, timeout=3)
eq_(3, len(events))
eq_(orig_endpoint, proxied_clt.current_endpoint)
# When
_fixture.disable_endpoint(orig_endpoint)
# Then: immediately reconnected to another endpoint.
_fixture.aux_clt().delete('/test/foo')
_fixture.aux_clt().put('/test/foo', 'bar5')
events = _collect_events(watch_queue, timeout=3)
eq_(2, len(events))
_assert_event(Event.DELETE, '/test/foo', '', events[0])
_assert_event(Event.PUT, '/test/foo', 'bar5', events[1])
assert_not_equal(orig_endpoint, proxied_clt.current_endpoint)
finally:
w.stop(timeout=1)
def test_auto_reconnect_outage():
# Connection is automatically restored even after a cluster wide outage.
proxied_clt = _fixture.proxied_clt()
proxied_clt.put('/test/foo', 'bar')
_assert_get_one('bar', proxied_clt.get('/test/foo'))
_fixture.disable_all_endpoints()
with _assert_raises_grpc_error(grpc.StatusCode.UNAVAILABLE,
'(OS Error)|'
'(Socket closed)|'
'(Transport closed)|'
'(Connect Failed)|'
'(failed to connect to all addresses)'):
proxied_clt.put('/test/foo', 'bazz1')
# When: etcd gets back in service
_fixture.enabled_all_endpoints()
# Then: client restores connection automatically
proxied_clt.put('/test/foo', 'bazz2')
_assert_get_one('bazz2', proxied_clt.get('/test/foo'))
def test_auto_reconnect_outage():
# If cluster wide outage lasts longer than the TTL, the key obviously
# expires, but it gets recreated after a connection is restored.
proxied_clt = _fixture.proxied_clt()
ttl = 2
keep_aliver = proxied_clt.new_keep_aliver('/test/keep-alive-2',
'bar',
ttl,
spin_pause=0.2)
keep_aliver.start()
try:
sleep(ttl - 0.5)
eq_(b'bar', proxied_clt.get_value('/test/keep-alive-2'))
sleep(ttl - 0.5)
eq_(b'bar', proxied_clt.get_value('/test/keep-alive-2'))
# If connection with Etcd is lost...
_fixture.disable_all_endpoints()
# ...the key expires
sleep(ttl + 0.5)
eq_(None, _fixture.aux_clt().get_value('/test/keep-alive-2'))
# When: etcd gets back in service
_fixture.enabled_all_endpoints()
# Then: the key is automatically restored.
sleep(2)
eq_(b'bar', proxied_clt.get_value('/test/keep-alive-2'))
sleep(ttl - 0.5)
eq_(b'bar', proxied_clt.get_value('/test/keep-alive-2'))
finally:
eq_(True, keep_aliver.stop(timeout=3))
eq_(None, proxied_clt.get_value('/test/keep-alive-2'))
def test_handler_exception():
# Event handler failures do not disrupt watch operation.
proxied_clt = _fixture.proxied_clt()
watch_queue = queue.Queue()
handle_count = [0]
def event_handler(e):
handle_count[0] += 1
if handle_count[0] == 2:
raise RuntimeError('Kaboom!')
watch_queue.put(e)
put_rs = _fixture.aux_clt().put('/test/foo', 'bar0')
w = proxied_clt.new_watcher('/test/foo',
spin_pause=0.2,
start_revision=put_rs.header.revision + 1,
event_handler=event_handler)
w.start()
try:
# When
proxied_clt.put('/test/foo', 'bar1')
proxied_clt.put('/test/foo', 'bar2')
proxied_clt.put('/test/foo', 'bar3')
proxied_clt.put('/test/foo', 'bar4')
# Then
events = _collect_events(watch_queue, timeout=3)
eq_(3, len(events))
_assert_event(Event.PUT, '/test/foo', 'bar1', events[0])
_assert_event(Event.PUT, '/test/foo', 'bar3', events[1])
_assert_event(Event.PUT, '/test/foo', 'bar4', events[2])
finally:
w.stop(timeout=1)
def test_auto_reconnect_node():
# If the node the client is connected to goes down, then the client
# recoonects to another node immediately.
proxied_clt = _fixture.proxied_clt()
ttl = 2
keep_aliver = proxied_clt.new_keep_aliver('/test/keep-alive-4',
'bar',
ttl,
spin_pause=0.2)
keep_aliver.start()
try:
sleep(ttl - 0.5)
eq_(b'bar', proxied_clt.get_value('/test/keep-alive-4'))
orig_endpoint = proxied_clt.current_endpoint
sleep(ttl - 0.5)
eq_(b'bar', proxied_clt.get_value('/test/keep-alive-4'))
eq_(orig_endpoint, proxied_clt.current_endpoint)
# When
_fixture.disable_endpoint(orig_endpoint)
sleep(ttl - 0.5)
eq_(b'bar', proxied_clt.get_value('/test/keep-alive-4'))
assert_not_equal(orig_endpoint, proxied_clt.current_endpoint)
sleep(ttl - 0.5)
eq_(b'bar', proxied_clt.get_value('/test/keep-alive-4'))
sleep(ttl - 0.5)
eq_(b'bar', proxied_clt.get_value('/test/keep-alive-4'))
finally:
eq_(True, keep_aliver.stop(timeout=3))
eq_(None, proxied_clt.get_value('/test/keep-alive-4'))
def test_etcd_down_on_start():
# If connection with Etcd cannot be established when a watch is started,
# then events are received as soon as the connection is up.
proxied_clt = _fixture.proxied_clt()
watch_queue = queue.Queue()
put_rs = proxied_clt.put('/test/foo', 'bar1')
proxied_clt.delete('/test/foo')
proxied_clt.put('/test/foo', 'bar2')
w = proxied_clt.new_watcher('/test/foo',
spin_pause=0.2,
start_revision=put_rs.header.revision,
event_handler=lambda e: watch_queue.put(e))
# When connection with Etcd is lost...
_fixture.disable_all_endpoints()
w.start()
try:
events = _collect_events(watch_queue, timeout=3)
eq_(0, len(events))
# But as soon as connection with Etcd is restored...
_fixture.enabled_all_endpoints()
# ...backed up events are received
events = _collect_events(watch_queue, timeout=5)
eq_(3, len(events))
_assert_event(Event.PUT, '/test/foo', 'bar1', events[0])
_assert_event(Event.DELETE, '/test/foo', '', events[1])
_assert_event(Event.PUT, '/test/foo', 'bar2', events[2])
finally:
w.stop(timeout=1)
