def test_failover(self):
self.log.info('\n\nStarting test_failover')
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
h = client.sendHellos(
memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO)
self.log.info(
'\n\nVerify responses are correct after graceful failover')
op_data = self.extract_vbucket_uuid_and_seqno(
client.set('failoverkey', 0, 0, 'failovervalue'))
op_data['format_type'] = 'no_failover'
# don't really need to do this so it is commented
#pre_failover_results = self.observe_seqno_response_to_dict( client.observe_seqno('failoverkey', vbucket_uuid) )
# which server did the key go to and gracefully fail that server
self.log.info('\n\nstarting graceful failover scenario')
server_with_key = client.memcached('failoverkey').host
self.log.info(
'\n\nserver {0} has the key and it will be failed over'.format(
server_with_key))
RebalanceHelper.wait_for_persistence(self.master,
self.default_bucket_name)
# now failover
RestConnection(self.master).fail_over(otpNode='ns_1@' +
server_with_key,
graceful=True)
if server_with_key in self.servers:
self.servers.remove(server_with_key)
self.log.info('server should be failed over now')
time.sleep(5)
# reinstantiate the client so we get the new view of the world
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
server_with_key = client.memcached('failoverkey').host
self.log.info('\n\nkey is now on server {0}'.format(server_with_key))
after_failover_results = self.observe_seqno_response_to_dict(
client.observe_seqno('failoverkey', op_data['vbucket_uuid']))
# verify: no (hard) failover, everything else as before
self.check_results(op_data, after_failover_results)
self.log.info('Test complete')
# now do a hard failover
# which server did the key go to and gracefully fail that server
time.sleep(30)
self.log.info('\n\nstarting hard failover scenario')
client.sendHellos(
memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO)
op_data = self.extract_vbucket_uuid_and_seqno(
client.set('hardfailoverkey', 0, 0, 'failovervalue'))
op_data['format_type'] = 'hard_failover'
server_with_key = client.memcached('hardfailoverkey').host
self.log.info(
'\n\nserver {0} has the key and it will be hard failed over'.
format(server_with_key))
# now failover
RestConnection(self.master).fail_over(otpNode='ns_1@' +
server_with_key,
graceful=False)
if server_with_key in self.servers:
self.servers.remove(server_with_key)
self.log.info('\n\nserver should be failed over now')
time.sleep(10)
# reinstantiate the client so we get the new view of the world
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
server_with_key = client.memcached('hardfailoverkey').host
self.log.info('\n\nkey is now on server {0}'.format(server_with_key))
time.sleep(10)
after_failover_results = self.observe_seqno_response_to_dict(
client.observe_seqno('hardfailoverkey', op_data['vbucket_uuid']))
self.check_results(op_data, after_failover_results)
self.log.info('Test complete')
def test_failover(self):
self.log.info('\n\nStarting test_failover')
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
h = client.sendHellos( memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO );
self.log.info('\n\nVerify responses are correct after graceful failover')
op_data = self.extract_vbucket_uuid_and_seqno( client.set('failoverkey', 0, 0, 'failovervalue') )
op_data['format_type'] = 'no_failover'
# don't really need to do this so it is commented
#pre_failover_results = self.observe_seqno_response_to_dict( client.observe_seqno('failoverkey', vbucket_uuid) )
# which server did the key go to and gracefully fail that server
self.log.info('\n\nstarting graceful failover scenario')
server_with_key = client.memcached( 'failoverkey').host
self.log.info('\n\nserver {0} has the key and it will be failed over'.format(server_with_key))
RebalanceHelper.wait_for_persistence(self.master, self.default_bucket_name)
# now failover
RestConnection(self.master).fail_over(otpNode = 'ns_1@' + server_with_key, graceful=True)
if server_with_key in self.servers:
self.servers.remove(server_with_key)
self.log.info('server should be failed over now')
time.sleep(5)
# reinstantiate the client so we get the new view of the world
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
server_with_key = client.memcached( 'failoverkey').host
self.log.info('\n\nkey is now on server {0}'.format(server_with_key))
after_failover_results = self.observe_seqno_response_to_dict(
client.observe_seqno('failoverkey', op_data['vbucket_uuid']) )
# verify: no (hard) failover, everything else as before
self.check_results( op_data, after_failover_results)
self.log.info('Test complete')
# now do a hard failover
# which server did the key go to and gracefully fail that server
self.log.info('\n\nstarting hard failover scenario')
client.sendHellos( memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO );
op_data = self.extract_vbucket_uuid_and_seqno( client.set('hardfailoverkey', 0, 0, 'failovervalue') )
op_data['format_type'] = 'hard_failover'
server_with_key = client.memcached( 'hardfailoverkey').host
self.log.info('\n\nserver {0} has the key and it will be hard failed over'.format(server_with_key))
# now failover
RestConnection(self.master).fail_over(otpNode = 'ns_1@' + server_with_key, graceful=False)
if server_with_key in self.servers:
self.servers.remove(server_with_key)
self.log.info('\n\nserver should be failed over now')
time.sleep(5)
# reinstantiate the client so we get the new view of the world
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
server_with_key = client.memcached( 'hardfailoverkey').host
self.log.info('\n\nkey is now on server {0}'.format(server_with_key))
after_failover_results = self.observe_seqno_response_to_dict(
client.observe_seqno('hardfailoverkey',op_data['vbucket_uuid']) )
self.check_results( op_data, after_failover_results)
self.log.info('Test complete')
def test_basic_operations(self):
self.log.info('\n\nStarting test_basic_operations')
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
h = client.sendHellos(
memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO)
all_clients = []
for s in self.servers:
all_clients.append(
MemcachedClientHelper.direct_client(s,
self.default_bucket_name))
# basic case
op_data = self.extract_vbucket_uuid_and_seqno(
client.set('test1key', 0, 0, 'test1value'))
op_data['format_type'] = 'no_failover'
for s in self.servers:
RebalanceHelper.wait_for_persistence(s, self.default_bucket_name)
o = client.observe_seqno('test1key', op_data['vbucket_uuid'])
results = self.observe_seqno_response_to_dict(o)
self.check_results(op_data, results)
# 2. Disable persistence, Set a key, observe seqno, should not be persisted,
# enable and wait for persistence, observe seqno and check everything
self.log.info(
'\n\nVerify responses are correct when keys are not persisted')
for i in all_clients:
i.stop_persistence()
mc = MemcachedClientHelper.direct_client(self.master, "default")
self.log.info('setting the kv')
op_data = self.extract_vbucket_uuid_and_seqno(
client.set('test2key', 0, 0, 'test2value'))
op_data['format_type'] = 'no_failover'
self.log.info('calling observe seq no')
o = client.observe_seqno('test2key', op_data['vbucket_uuid'])
results = self.observe_seqno_response_to_dict(o)
# can't use check results because persisted is
self.assertTrue(
op_data['vbucket_uuid'] == results['new_vbucket_uuid'],
msg='Observe Vbucket uuid does not match. Expected: {0}. Actual {1}'
.format(hex(op_data['vbucket_uuid']),
hex(results['new_vbucket_uuid'])))
self.assertTrue(
op_data['seqno'] == results['current_seqno'],
msg='Observe seqno does not match. Expected: {0}. Actual {1}'.
format(op_data['seqno'], results['current_seqno']))
self.assertTrue(
op_data['seqno'] > results['last_persisted_seqno'],
msg='Persisted seqno is too big. Expected: {0}. Actual {1}'.format(
op_data['seqno'], results['last_persisted_seqno']))
self.log.info('starting persistence')
for s in all_clients:
s.start_persistence()
for s in self.servers:
RebalanceHelper.wait_for_persistence(s, self.default_bucket_name)
results = self.observe_seqno_response_to_dict(
client.observe_seqno('test2key', op_data['vbucket_uuid']))
self.check_results(op_data, results)
# error case - broken
"""
mc.set('badbuckettestkey', 0, 0, 'testvalue',1)
try:
o = client.observe_seqno('badbuckettestkey', 2)
self.fail('bucket is incorrect, should have returned an error')
except AssertionError, ex: # this is to catch the above fail, should it ever happen
raise
except Exception, ex:
traceback.print_exc()
if ex.status != memcacheConstants.ERR_NOT_FOUND:
self.log.info('Observe seqno incorrect error code for invalid bucket. Expected: {0}. Actual {1}'.format(
memcacheConstants.ERR_NOT_FOUND, ex.status))
raise Exception(ex)
"""
self.log.info('\n\nComplete test_basic_operations')
def test_basic_operations(self):
self.log.info('\n\nStarting test_basic_operations')
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
h = client.sendHellos( memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO );
all_clients = []
for s in self.servers:
all_clients.append( MemcachedClientHelper.direct_client(s, self.default_bucket_name))
# basic case
op_data = self.extract_vbucket_uuid_and_seqno( client.set('test1key', 0, 0, 'test1value') )
op_data['format_type'] = 'no_failover'
for s in self.servers:
RebalanceHelper.wait_for_persistence(s, self.default_bucket_name)
o = client.observe_seqno('test1key', op_data['vbucket_uuid'])
results = self.observe_seqno_response_to_dict( o )
self.check_results( op_data, results)
# 2. Disable persistence, Set a key, observe seqno, should not be persisted,
# enable and wait for persistence, observe seqno and check everything
self.log.info('\n\nVerify responses are correct when keys are not persisted')
for i in all_clients:
i.stop_persistence()
mc = MemcachedClientHelper.direct_client(self.master, "default")
self.log.info('setting the kv')
op_data = self.extract_vbucket_uuid_and_seqno( client.set('test2key', 0, 0, 'test2value') )
op_data['format_type'] = 'no_failover'
self.log.info('calling observe seq no')
o = client.observe_seqno('test2key', op_data['vbucket_uuid'])
results = self.observe_seqno_response_to_dict( o )
# can't use check results because persisted is
self.assertTrue(op_data['vbucket_uuid'] == results['new_vbucket_uuid'],
msg='Observe Vbucket uuid does not match. Expected: {0}. Actual {1}'.format(
hex(op_data['vbucket_uuid']), hex(results['new_vbucket_uuid'])) )
self.assertTrue(op_data['seqno'] == results['current_seqno'],
msg='Observe seqno does not match. Expected: {0}. Actual {1}'.format(
op_data['seqno'], results['current_seqno']) )
self.assertTrue(op_data['seqno'] > results['last_persisted_seqno'],
msg='Persisted seqno is too big. Expected: {0}. Actual {1}'.format(
op_data['seqno'], results['last_persisted_seqno']) )
self.log.info('starting persistence')
for s in all_clients:
s.start_persistence()
for s in self.servers:
RebalanceHelper.wait_for_persistence(s, self.default_bucket_name)
results = self.observe_seqno_response_to_dict( client.observe_seqno('test2key', op_data['vbucket_uuid']) )
self.check_results( op_data, results)
# error case
mc.set('badbuckettestkey', 0, 0, 'testvalue',1)
try:
o = client.observe_seqno('badbuckettestkey', 2)
self.fail('bucket is incorrect, should have returned an error')
except AssertionError, ex: # this is to catch the above fail, should it ever happen
raise
