def test_auto_reconnect_node(): # If a connection with current node is lost then the client automatically # connects to another. proxied_clt = _fixture.proxied_clt() proxied_clt.put('/test/foo', 'bar') _assert_get_one('bar', proxied_clt.get('/test/foo')) orig_endpoint = proxied_clt.current_endpoint i = 0 while True: val = 'bazz%d' % (i,) endpoint = proxied_clt.current_endpoint # When _fixture.disable_endpoint(endpoint) try: # Then proxied_clt.put('/test/foo', val) assert_not_equal(endpoint, proxied_clt.current_endpoint) _assert_get_one(val, proxied_clt.get('/test/foo')) finally: _fixture.enable_endpoint(endpoint) # Eventually we should cycle through all endpoints and get back to the # origin. if endpoint == orig_endpoint: break
def test_watch_from_the_past(): # A revision to start watching from can be specified. proxied_clt = _fixture.proxied_clt() watch_queue = queue.Queue() proxied_clt.put('/test/foo2', 'bar1') proxied_clt.put('/test/foo2', 'bar2') put_rs = proxied_clt.put('/test/foo2', 'bar3') proxied_clt.put('/test/foo2', 'bar4') proxied_clt.put('/test/foo2', 'bar5') proxied_clt.put('/test/foo3', 'bar6') # When w = proxied_clt.new_watcher('/test/foo2', spin_pause=0.2, start_revision=put_rs.header.revision, event_handler=lambda e: watch_queue.put(e)) w.start() try: proxied_clt.delete('/test/foo2') # Then events = _collect_events(watch_queue, timeout=3) eq_(4, len(events)) _assert_event(Event.PUT, '/test/foo2', 'bar3', events[0]) _assert_event(Event.PUT, '/test/foo2', 'bar4', events[1]) _assert_event(Event.PUT, '/test/foo2', 'bar5', events[2]) _assert_event(Event.DELETE, '/test/foo2', '', events[3]) finally: w.stop(timeout=1)
def test_watch_key_prefix(): proxied_clt = _fixture.proxied_clt() watch_queue = queue.Queue() proxied_clt.put('/test/foo2', 'bar1') w = proxied_clt.new_watcher('/test/foo2', is_prefix=True, spin_pause=0.2, event_handler=lambda e: watch_queue.put(e)) w.start() try: sleep(0.5) # When proxied_clt.put('/test/foo1', 'bar6') proxied_clt.put('/test/foo21', 'bar7') proxied_clt.delete('/test/foo2') proxied_clt.put('/test/foo212', 'bar8') proxied_clt.delete('/test/foo1') proxied_clt.put('/test/foo21', 'bar10') proxied_clt.put('/test/foo3', 'bar11') # Then events = _collect_events(watch_queue, timeout=3) eq_(4, len(events)) _assert_event(Event.PUT, '/test/foo21', 'bar7', events[0]) _assert_event(Event.DELETE, '/test/foo2', '', events[1]) _assert_event(Event.PUT, '/test/foo212', 'bar8', events[2]) _assert_event(Event.PUT, '/test/foo21', 'bar10', events[3]) finally: w.stop(timeout=1)
def test_etcd_down_on_start(): # It is ok to create a keep_aliver when connection with Etcd is down. The # key will be created as soon as connection is restored. proxied_clt = _fixture.proxied_clt() # Simulate Etcd down on start. _fixture.disable_all_endpoints() ttl = 2 keep_aliver = proxied_clt.new_keep_aliver('/test/keep-alive-3', 'bar', ttl, spin_pause=0.2) keep_aliver.start() try: sleep(0.5) eq_(None, _fixture.aux_clt().get_value('/test/keep-alive-3')) # When: etcd gets back in service. _fixture.enabled_all_endpoints() # Then: the key is automatically created. sleep(0.5) eq_(b'bar', proxied_clt.get_value('/test/keep-alive-3')) sleep(ttl - 0.5) eq_(b'bar', proxied_clt.get_value('/test/keep-alive-3')) finally: eq_(True, keep_aliver.stop(timeout=3)) eq_(None, proxied_clt.get_value('/test/keep-alive-3'))
def test_get_does_not_exist(): proxied_clt = _fixture.proxied_clt() # When rs = proxied_clt.get('/test/foo') # Then _assert_get_one(None, rs)
def test_lease_revoke(): proxied_clt = _fixture.proxied_clt() rs = proxied_clt.lease_grant(600) proxied_clt.put('/test/foo', 'bar', rs.ID) _assert_get_one('bar', proxied_clt.get('/test/foo')) # When proxied_clt.lease_revoke(rs.ID) # Then _assert_get_one(None, proxied_clt.get('/test/foo'))
def test_lease_expires(): proxied_clt = _fixture.proxied_clt() rs = proxied_clt.lease_grant(1) proxied_clt.put('/test/foo', 'bar', rs.ID) _assert_get_one('bar', proxied_clt.get('/test/foo')) # When sleep(rs.TTL + 1) # Then _assert_get_one(None, proxied_clt.get('/test/foo'))
def test_get_prefix_limit(): proxied_clt = _fixture.proxied_clt() proxied_clt.put('/test/foo1', 'bar5') proxied_clt.put('/test/foo2', 'bar4') proxied_clt.put('/test/foo21', 'bar3') proxied_clt.put('/test/foo212', 'bar2') proxied_clt.put('/test/foo3', 'bar1') # When rs = proxied_clt.get('/test/foo', is_prefix=True, limit=2) # Then eq_(5, rs.count) eq_(2, len(rs.kvs))
def test_get_prefix(): proxied_clt = _fixture.proxied_clt() proxied_clt.put('/test/foo1', 'bar1') proxied_clt.put('/test/foo2', 'bar2') proxied_clt.put('/test/foo21', 'bar3') proxied_clt.put('/test/foo212', 'bar4') proxied_clt.put('/test/foo3', 'bar5') # When rs = proxied_clt.get('/test/foo2', is_prefix=True) # Then eq_(3, rs.count) eq_(3, len(rs.kvs)) eq_(b'bar2', rs.kvs[0].value) eq_(b'bar3', rs.kvs[1].value) eq_(b'bar4', rs.kvs[2].value)
def test_delete_range(): proxied_clt = _fixture.proxied_clt() proxied_clt.put('/test/foo1', 'bar1') proxied_clt.put('/test/foo2', 'bar2') proxied_clt.put('/test/foo21', 'bar3') proxied_clt.put('/test/foo212', 'bar4') proxied_clt.put('/test/foo3', 'bar5') # When rs = proxied_clt.delete('/test/foo2', is_prefix=True) # Then eq_(3, rs.deleted) _assert_get_one('bar1', proxied_clt.get('/test/foo1')) _assert_get_one(None, proxied_clt.get('/test/foo2')) _assert_get_one(None, proxied_clt.get('/test/foo21')) _assert_get_one(None, proxied_clt.get('/test/foo212')) _assert_get_one('bar5', proxied_clt.get('/test/foo3'))
def test_spin_pause_too_long(): # Spin pause is adjusted if needed to be less then effective TTL. proxied_clt = _fixture.proxied_clt() ttl = 3 keep_aliver = proxied_clt.new_keep_aliver('/test/keep-alive-1', 'bar', ttl, spin_pause=10) keep_aliver.start() try: sleep(ttl - 0.5) eq_(b'bar', proxied_clt.get_value('/test/keep-alive-1')) finally: eq_(True, keep_aliver.stop(timeout=3)) eq_(None, proxied_clt.get_value('/test/keep-alive-1'))
def test_get_revisions(): # Checks that create_revision and mod_revision are returned correctly. proxied_clt = _fixture.proxied_clt() # Given proxied_clt.put('/test/foo0', 'tox') create_rs = proxied_clt.put('/test/foo', 'bar') proxied_clt.put('/test/foo', 'bazz') proxied_clt.put('/test/foo2', 'fox') update_rs = proxied_clt.put('/test/foo', 'blah') proxied_clt.put('/test/foo3', 'socks') # When rs = proxied_clt.get('/test/foo') # Then eq_(create_rs.header.revision, rs.kvs[0].create_revision) eq_(update_rs.header.revision, rs.kvs[0].mod_revision) _assert_get_one('blah', rs)
def test_normal_operation(): # KeepAliver ensure that key exists while running and deletes it on stop. proxied_clt = _fixture.proxied_clt() ttl = 2 keep_aliver = proxied_clt.new_keep_aliver('/test/keep-alive-0', 'bar', ttl, spin_pause=0.2) keep_aliver.start() try: for i in range(3): sleep(ttl - 0.5) eq_(b'bar', proxied_clt.get_value('/test/keep-alive-0')) finally: eq_(True, keep_aliver.stop(timeout=3)) eq_(None, proxied_clt.get_value('/test/keep-alive-0'))
def test_auto_reconnect_outage(): # Watch is restored even after a cluster wide outage. proxied_clt = _fixture.proxied_clt() watch_queue = queue.Queue() put_rs = _fixture.aux_clt().put('/test/foo', 'bar0') w = proxied_clt.new_watcher('/test/foo', spin_pause=0.2, start_revision=put_rs.header.revision + 1, event_handler=lambda e: watch_queue.put(e)) w.start() try: _fixture.aux_clt().put('/test/foo', 'bar1') _fixture.aux_clt().put('/test/foo', 'bar2') sleep(0.5) # When connection with Etcd is lost... _fixture.disable_all_endpoints() # New events stop coming. _fixture.aux_clt().put('/test/foo', 'bar3') _fixture.aux_clt().delete('/test/foo') _fixture.aux_clt().put('/test/foo', 'bar4') events = _collect_events(watch_queue, timeout=3) eq_(2, len(events)) _assert_event(Event.PUT, '/test/foo', 'bar1', events[0]) _assert_event(Event.PUT, '/test/foo', 'bar2', events[1]) # But as soon as the cluster is back in service... _fixture.enabled_all_endpoints() # ...backed up events are received events = _collect_events(watch_queue, timeout=5) eq_(3, len(events)) _assert_event(Event.PUT, '/test/foo', 'bar3', events[0]) _assert_event(Event.DELETE, '/test/foo', '', events[1]) _assert_event(Event.PUT, '/test/foo', 'bar4', events[2]) finally: w.stop(timeout=1)
def test_get_prefix_sort(): proxied_clt = _fixture.proxied_clt() proxied_clt.put('/test/foo212', 'bar4') proxied_clt.put('/test/foo2', 'bar2') proxied_clt.put('/test/foo3', 'bar1') proxied_clt.put('/test/foo1', 'bar5') proxied_clt.put('/test/foo21', 'bar3') for i, tc in enumerate([{ 'order': SortOrder.ASCEND, 'target': SortTarget.KEY, 'out': [b'bar5', b'bar2', b'bar3', b'bar4', b'bar1'], }, { 'order': SortOrder.ASCEND, 'target': SortTarget.VALUE, 'out': [b'bar1', b'bar2', b'bar3', b'bar4', b'bar5'], }, { 'order': SortOrder.ASCEND, 'target': SortTarget.CREATE_REVISION, 'out': [b'bar4', b'bar2', b'bar1', b'bar5', b'bar3'], }, { 'order': SortOrder.DESCEND, 'target': SortTarget.KEY, 'out': [b'bar1', b'bar4', b'bar3', b'bar2', b'bar5'], }, { 'order': SortOrder.DESCEND, 'target': SortTarget.VALUE, 'out': [b'bar5', b'bar4', b'bar3', b'bar2', b'bar1'], }, { 'order': SortOrder.DESCEND, 'target': SortTarget.CREATE_REVISION, 'out': [b'bar3', b'bar5', b'bar1', b'bar2', b'bar4'], }]): print('Test case #%d: %s/%s', i, tc['order'], tc['target']) # When rs = proxied_clt.get('/test/foo', is_prefix=True, sort_order=tc['order'], sort_target=tc['target']) # Then eq_(tc['out'], [kv.value for kv in rs.kvs])
def test_auto_reconnect_node(): # Watch is restored even after a cluster wide outage. proxied_clt = _fixture.proxied_clt() watch_queue = queue.Queue() put_rs = _fixture.aux_clt().put('/test/foo', 'bar0') w = proxied_clt.new_watcher('/test/foo', spin_pause=0.2, start_revision=put_rs.header.revision + 1, event_handler=lambda e: watch_queue.put(e)) w.start() try: _fixture.aux_clt().put('/test/foo', 'bar1') events = _collect_events(watch_queue, timeout=3) eq_(1, len(events)) orig_endpoint = proxied_clt.current_endpoint _fixture.aux_clt().put('/test/foo', 'bar2') _fixture.aux_clt().put('/test/foo', 'bar3') _fixture.aux_clt().put('/test/foo', 'bar4') events = _collect_events(watch_queue, timeout=3) eq_(3, len(events)) eq_(orig_endpoint, proxied_clt.current_endpoint) # When _fixture.disable_endpoint(orig_endpoint) # Then: immediately reconnected to another endpoint. _fixture.aux_clt().delete('/test/foo') _fixture.aux_clt().put('/test/foo', 'bar5') events = _collect_events(watch_queue, timeout=3) eq_(2, len(events)) _assert_event(Event.DELETE, '/test/foo', '', events[0]) _assert_event(Event.PUT, '/test/foo', 'bar5', events[1]) assert_not_equal(orig_endpoint, proxied_clt.current_endpoint) finally: w.stop(timeout=1)
def test_auto_reconnect_outage(): # Connection is automatically restored even after a cluster wide outage. proxied_clt = _fixture.proxied_clt() proxied_clt.put('/test/foo', 'bar') _assert_get_one('bar', proxied_clt.get('/test/foo')) _fixture.disable_all_endpoints() with _assert_raises_grpc_error(grpc.StatusCode.UNAVAILABLE, '(OS Error)|' '(Socket closed)|' '(Transport closed)|' '(Connect Failed)|' '(failed to connect to all addresses)'): proxied_clt.put('/test/foo', 'bazz1') # When: etcd gets back in service _fixture.enabled_all_endpoints() # Then: client restores connection automatically proxied_clt.put('/test/foo', 'bazz2') _assert_get_one('bazz2', proxied_clt.get('/test/foo'))
def test_auto_reconnect_outage(): # If cluster wide outage lasts longer than the TTL, the key obviously # expires, but it gets recreated after a connection is restored. proxied_clt = _fixture.proxied_clt() ttl = 2 keep_aliver = proxied_clt.new_keep_aliver('/test/keep-alive-2', 'bar', ttl, spin_pause=0.2) keep_aliver.start() try: sleep(ttl - 0.5) eq_(b'bar', proxied_clt.get_value('/test/keep-alive-2')) sleep(ttl - 0.5) eq_(b'bar', proxied_clt.get_value('/test/keep-alive-2')) # If connection with Etcd is lost... _fixture.disable_all_endpoints() # ...the key expires sleep(ttl + 0.5) eq_(None, _fixture.aux_clt().get_value('/test/keep-alive-2')) # When: etcd gets back in service _fixture.enabled_all_endpoints() # Then: the key is automatically restored. sleep(2) eq_(b'bar', proxied_clt.get_value('/test/keep-alive-2')) sleep(ttl - 0.5) eq_(b'bar', proxied_clt.get_value('/test/keep-alive-2')) finally: eq_(True, keep_aliver.stop(timeout=3)) eq_(None, proxied_clt.get_value('/test/keep-alive-2'))
def test_handler_exception(): # Event handler failures do not disrupt watch operation. proxied_clt = _fixture.proxied_clt() watch_queue = queue.Queue() handle_count = [0] def event_handler(e): handle_count[0] += 1 if handle_count[0] == 2: raise RuntimeError('Kaboom!') watch_queue.put(e) put_rs = _fixture.aux_clt().put('/test/foo', 'bar0') w = proxied_clt.new_watcher('/test/foo', spin_pause=0.2, start_revision=put_rs.header.revision + 1, event_handler=event_handler) w.start() try: # When proxied_clt.put('/test/foo', 'bar1') proxied_clt.put('/test/foo', 'bar2') proxied_clt.put('/test/foo', 'bar3') proxied_clt.put('/test/foo', 'bar4') # Then events = _collect_events(watch_queue, timeout=3) eq_(3, len(events)) _assert_event(Event.PUT, '/test/foo', 'bar1', events[0]) _assert_event(Event.PUT, '/test/foo', 'bar3', events[1]) _assert_event(Event.PUT, '/test/foo', 'bar4', events[2]) finally: w.stop(timeout=1)
def test_auto_reconnect_node(): # If the node the client is connected to goes down, then the client # recoonects to another node immediately. proxied_clt = _fixture.proxied_clt() ttl = 2 keep_aliver = proxied_clt.new_keep_aliver('/test/keep-alive-4', 'bar', ttl, spin_pause=0.2) keep_aliver.start() try: sleep(ttl - 0.5) eq_(b'bar', proxied_clt.get_value('/test/keep-alive-4')) orig_endpoint = proxied_clt.current_endpoint sleep(ttl - 0.5) eq_(b'bar', proxied_clt.get_value('/test/keep-alive-4')) eq_(orig_endpoint, proxied_clt.current_endpoint) # When _fixture.disable_endpoint(orig_endpoint) sleep(ttl - 0.5) eq_(b'bar', proxied_clt.get_value('/test/keep-alive-4')) assert_not_equal(orig_endpoint, proxied_clt.current_endpoint) sleep(ttl - 0.5) eq_(b'bar', proxied_clt.get_value('/test/keep-alive-4')) sleep(ttl - 0.5) eq_(b'bar', proxied_clt.get_value('/test/keep-alive-4')) finally: eq_(True, keep_aliver.stop(timeout=3)) eq_(None, proxied_clt.get_value('/test/keep-alive-4'))
def test_etcd_down_on_start(): # If connection with Etcd cannot be established when a watch is started, # then events are received as soon as the connection is up. proxied_clt = _fixture.proxied_clt() watch_queue = queue.Queue() put_rs = proxied_clt.put('/test/foo', 'bar1') proxied_clt.delete('/test/foo') proxied_clt.put('/test/foo', 'bar2') w = proxied_clt.new_watcher('/test/foo', spin_pause=0.2, start_revision=put_rs.header.revision, event_handler=lambda e: watch_queue.put(e)) # When connection with Etcd is lost... _fixture.disable_all_endpoints() w.start() try: events = _collect_events(watch_queue, timeout=3) eq_(0, len(events)) # But as soon as connection with Etcd is restored... _fixture.enabled_all_endpoints() # ...backed up events are received events = _collect_events(watch_queue, timeout=5) eq_(3, len(events)) _assert_event(Event.PUT, '/test/foo', 'bar1', events[0]) _assert_event(Event.DELETE, '/test/foo', '', events[1]) _assert_event(Event.PUT, '/test/foo', 'bar2', events[2]) finally: w.stop(timeout=1)