def simple_handshake_demo():
upr_client = UprClient('127.0.0.1', 12000)
mcd_client = McdClient('127.0.0.1', 12000)
op = upr_client.open_producer("mystream")
print 'Sending open connection (producer)'
print op
response = op.next_response()
print 'Response: %s\n' % response
assert response['status'] == SUCCESS
op = upr_client.stream_req(0, 0, 0, 0, 0, 0)
print 'Sending Stream Request'
print op
while op.has_response():
response = op.next_response()
print 'Response: %s' % response
assert response['status'] == SUCCESS
print '\nGet Tap Stats\n'
op = mcd_client.stats('tap')
response = op.next_response()
pprint.pprint(response['value'])
upr_client.shutdown()
print '\n'
print 'Closed Upr Connection'
print '\nGet Tap Stats\n'
op = mcd_client.stats('tap')
response = op.next_response()
pprint.pprint(response['value'])
mcd_client.shutdown()
def wait_for_warmup(host, port):
while True:
client = McdClient(host, port)
op = client.stats()
response = op.next_response()
if response["status"] == SUCCESS:
if response["value"]["ep_degraded_mode"] == "0":
break
time.sleep(1)
class McdTestCase(ParametrizedTestCase):
def setUp(self):
self.client = McdClient(self.host, self.port)
if self.backend == RemoteServer.MCD:
resp = self.client.flush().next_response()
assert resp["status"] == SUCCESS, "Flush all is not enabled %s" % resp
def tearDown(self):
self.client.shutdown()
def wait_for_stat(self, stat, val, type=""):
for i in range(5):
op = self.client.stats(type)
resp = op.next_response()
assert resp["status"] == SUCCESS
if resp["value"][stat] == str(val):
return True
time.sleep(1)
return False
def test_stats(self):
op = self.client.stats()
resp = op.next_response()
assert resp["status"] == SUCCESS
assert resp["value"]["curr_items"] == "0"
def test_stats_tap(self):
op = self.client.stats("tap")
resp = op.next_response()
assert resp["status"] == SUCCESS
assert resp["value"]["ep_tap_backoff_period"] == "5"
@skipUnlessMcd
def test_set(self):
op = self.client.set("key", "value", 0, 0, 0)
resp = op.next_response()
op = self.client.stats()
resp = op.next_response()
assert resp["status"] == SUCCESS
assert resp["value"]["curr_items"] == "1"
@skipUnlessMcd
def test_delete(self):
op = self.client.set("key1", "value", 0, 0, 0)
resp = op.next_response()
assert resp["status"] == SUCCESS
op = self.client.delete("key1", 0)
resp = op.next_response()
assert resp["status"] == SUCCESS
assert self.wait_for_stat("curr_items", 0)
def multiple_streams(host, port):
upr_client = UprClient(host, port)
mcd_client = McdClient(host, port)
op = upr_client.sasl_auth_plain('gamesim-sample', '')
response = op.next_response()
assert response['status'] == SUCCESS
op = mcd_client.sasl_auth_plain('gamesim-sample', '')
response = op.next_response()
assert response['status'] == SUCCESS
num_vbs = 10
op = mcd_client.stats('vbucket-seqno')
resp = op.next_response()
assert resp['status'] == SUCCESS
op = upr_client.open_producer("mystream")
response = op.next_response()
assert response['status'] == SUCCESS
streams = {}
for vb in range(num_vbs):
en = int(resp['value']['vb_%d_high_seqno' % vb])
op = upr_client.stream_req(vb, 0, 0, en, 0, 0)
print "Create stream vb %d st 0 en %d" % (vb, en)
streams[vb] = {'op' : op,
'mutations' : 0,
'last_seqno' : 0 }
while len(streams) > 0:
for vb in streams.keys():
if streams[vb]['op'].has_response():
response = streams[vb]['op'].next_response()
if response['opcode'] == CMD_STREAM_REQ:
assert response['status'] == SUCCESS
elif response['opcode'] == CMD_SNAPSHOT_MARKER:
pass
elif response['opcode'] == CMD_MUTATION:
assert response['by_seqno'] > streams[vb]['last_seqno']
streams[vb]['last_seqno'] = response['by_seqno']
streams[vb]['mutations'] = streams[vb]['mutations'] + 1
vb = response['vbucket']
key = response['key']
seqno = response['by_seqno']
print 'VB: %d got key %s with seqno %d' % (vb, key, seqno)
else:
del streams[vb]
upr_client.shutdown()
mcd_client.shutdown()
def couchbase_backend_setup(self):
self.rest_client = RestClient(self.host, port=self.rest_port)
for bucket in self.rest_client.get_all_buckets():
logging.info("Deleting bucket %s" % bucket)
assert self.rest_client.delete_bucket(bucket)
logging.info("Creating default bucket")
assert self.rest_client.create_default_bucket()
Stats.wait_for_warmup(self.host, self.port)
self.upr_client = UprClient(self.host, self.port)
self.mcd_client = McdClient(self.host, self.port)
def add_stream_demo():
upr_client = UprClient('127.0.0.1', 12000)
mcd_client = McdClient('127.0.0.1', 12000)
op = upr_client.open_consumer("mystream")
print 'Sending open connection (consumer)'
print op
response = op.next_response()
print 'Response: %s\n' % response
assert response['status'] == SUCCESS
op = upr_client.add_stream(0, 0)
print 'Sending add stream request'
print op
response = op.next_response()
assert response['status'] == SUCCESS
print 'Got add stream response'
print response
upr_client.shutdown()
mcd_client.shutdown()
def add_stream_demo():
upr_client = UprClient('127.0.0.1', 12000)
mcd_client = McdClient('127.0.0.1', 12000)
op = upr_client.open_consumer("mystream")
print 'Sending open connection (consumer)'
print op
response = op.next_response()
print 'Response: %s\n' % response
assert response['status'] == SUCCESS
op = upr_client.add_stream(0, 0)
print 'Sending add stream request'
print op
response = op.next_response()
assert response['status'] == SUCCESS
print 'Got add stream response'
print response
upr_client.shutdown()
mcd_client.shutdown()
def simple_handshake_demo():
upr_client = UprClient('127.0.0.1', 12000)
mcd_client = McdClient('127.0.0.1', 12000)
op = upr_client.open_producer("mystream")
print 'Sending open connection (producer)'
print op
response = op.next_response()
print 'Response: %s\n' % response
assert response['status'] == SUCCESS
op = upr_client.stream_req(0, 0, 0, 0, 0, 0)
print 'Sending Stream Request'
print op
while op.has_response():
response = op.next_response()
print 'Response: %s' % response
assert response['status'] == SUCCESS
print '\nGet Tap Stats\n'
op = mcd_client.stats('tap')
response = op.next_response()
pprint.pprint(response['value'])
upr_client.shutdown()
print '\n'
print 'Closed Upr Connection'
print '\nGet Tap Stats\n'
op = mcd_client.stats('tap')
response = op.next_response()
pprint.pprint(response['value'])
mcd_client.shutdown()
def two_way_demo():
consumer = DcpClient('127.0.0.1', 5000)
producer = DcpClient('127.0.0.1', 5001)
consumer.set_proxy(producer)
producer.set_proxy(consumer)
mcd_consumer = McdClient('127.0.0.1', 5000)
mcd_producer = McdClient('127.0.0.1', 5001)
response = get_resp(consumer.open_consumer("mystream"), 'Sending open connection (consumer)')
assert response['status'] == SUCCESS
response = get_resp(producer.open_producer("mystream"), 'Sending open connection (producer)')
assert response['status'] == SUCCESS
response = get_resp(consumer.add_stream(0, 0), 'Sending add stream request for vbucket 0')
assert response['status'] == SUCCESS
op = mcd_consumer.stats('upr')
response = op.next_response()
print response['value']
assert response['value']['ep_upr_count'] == '1'
op = mcd_producer.stats('upr')
response = op.next_response()
print response['value']
assert response['value']['ep_upr_count'] == '1'
get_resp(mcd_producer.set("key", "blah", 0, 0, 0), 'Create mutation on producer')
time.sleep(3)
get_resp(mcd_producer.delete("key", 0), 'Create mutation on producer')
time.sleep(10)
consumer.shutdown()
producer.shutdown()
mcd_consumer.shutdown()
mcd_producer.shutdown()
def multiple_streams(host, port):
upr_client = UprClient(host, port)
mcd_client = McdClient(host, port)
op = upr_client.sasl_auth_plain('gamesim-sample', '')
response = op.next_response()
assert response['status'] == SUCCESS
op = mcd_client.sasl_auth_plain('gamesim-sample', '')
response = op.next_response()
assert response['status'] == SUCCESS
num_vbs = 10
op = mcd_client.stats('vbucket-seqno')
resp = op.next_response()
assert resp['status'] == SUCCESS
op = upr_client.open_producer("mystream")
response = op.next_response()
assert response['status'] == SUCCESS
streams = {}
for vb in range(num_vbs):
en = int(resp['value']['vb_%d_high_seqno' % vb])
op = upr_client.stream_req(vb, 0, 0, en, 0, 0)
print "Create stream vb %d st 0 en %d" % (vb, en)
streams[vb] = {'op' : op,
'mutations' : 0,
'last_seqno' : 0 }
while len(streams) > 0:
for vb in streams.keys():
if streams[vb]['op'].has_response():
response = streams[vb]['op'].next_response()
if response['opcode'] == CMD_STREAM_REQ:
assert response['status'] == SUCCESS
elif response['opcode'] == CMD_SNAPSHOT_MARKER:
pass
elif response['opcode'] == CMD_MUTATION:
assert response['by_seqno'] > streams[vb]['last_seqno']
streams[vb]['last_seqno'] = response['by_seqno']
streams[vb]['mutations'] = streams[vb]['mutations'] + 1
vb = response['vbucket']
key = response['key']
seqno = response['by_seqno']
print 'VB: %d got key %s with seqno %d' % (vb, key, seqno)
else:
del streams[vb]
upr_client.shutdown()
mcd_client.shutdown()
def memcached_backend_setup(self):
self.upr_client = UprClient(self.host, self.port)
self.mcd_client = McdClient(self.host, self.port)
resp = self.mcd_client.flush().next_response()
assert resp['status'] == SUCCESS, "Flush all is not enabled"
class ParametrizedTestCase(unittest.TestCase):
""" TestCase classes that want to be parametrized should
inherit from this class.
"""
def __init__(self, methodName, backend, host, port):
super(ParametrizedTestCase, self).__init__(methodName)
self.backend = backend
self.host = host
self.port = port
if host.find(':') != -1:
self.host, self.rest_port = host.split(':')
else:
self.rest_port = 9000
def initialize_backend(self):
print ''
logging.info("-------Setup Test Case-------")
self.rest_client = RestClient(self.host, port=self.rest_port)
if (self.backend == RemoteServer.MCD):
self.memcached_backend_setup()
else:
self.couchbase_backend_setup()
logging.info("-----Begin Test Case-----")
def destroy_backend(self):
logging.info("-----Tear Down Test Case-----")
if (self.backend == RemoteServer.MCD):
self.memcached_backend_teardown()
else:
self.couchbase_backend_teardown()
def memcached_backend_setup(self):
self.upr_client = UprClient(self.host, self.port)
self.mcd_client = McdClient(self.host, self.port)
resp = self.mcd_client.flush().next_response()
assert resp['status'] == SUCCESS, "Flush all is not enabled"
def memcached_backend_teardown(self):
self.upr_client.shutdown()
self.mcd_client.shutdown()
def couchbase_backend_setup(self):
self.rest_client = RestClient(self.host, port=self.rest_port)
for bucket in self.rest_client.get_all_buckets():
logging.info("Deleting bucket %s" % bucket)
assert self.rest_client.delete_bucket(bucket)
logging.info("Creating default bucket")
assert self.rest_client.create_default_bucket()
Stats.wait_for_warmup(self.host, self.port)
self.upr_client = UprClient(self.host, self.port)
self.mcd_client = McdClient(self.host, self.port)
def couchbase_backend_teardown(self):
self.upr_client.shutdown()
self.mcd_client.shutdown()
for bucket in self.rest_client.get_all_buckets():
logging.info("Deleting bucket %s" % bucket)
assert self.rest_client.delete_bucket(bucket)
self.rest_client = None
@staticmethod
def parametrize(testcase_klass, backend, host, port):
""" Create a suite containing all tests taken from the given
subclass, passing them the parameter 'param'.
"""
testloader = unittest.TestLoader()
testnames = testloader.getTestCaseNames(testcase_klass)
suite = unittest.TestSuite()
for name in testnames:
suite.addTest(testcase_klass(name, backend, host, port))
return suite
def two_way_demo():
consumer = UprClient('127.0.0.1', 5000)
producer = UprClient('127.0.0.1', 5001)
consumer.set_proxy(producer)
producer.set_proxy(consumer)
mcd_consumer = McdClient('127.0.0.1', 5000)
mcd_producer = McdClient('127.0.0.1', 5001)
response = get_resp(consumer.open_consumer("mystream"), 'Sending open connection (consumer)')
assert response['status'] == SUCCESS
response = get_resp(producer.open_producer("mystream"), 'Sending open connection (producer)')
assert response['status'] == SUCCESS
response = get_resp(consumer.add_stream(0, 0), 'Sending add stream request for vbucket 0')
assert response['status'] == SUCCESS
op = mcd_consumer.stats('upr')
response = op.next_response()
print response['value']
assert response['value']['ep_upr_count'] == '1'
op = mcd_producer.stats('upr')
response = op.next_response()
print response['value']
assert response['value']['ep_upr_count'] == '1'
get_resp(mcd_producer.set("key", "blah", 0, 0, 0), 'Create mutation on producer')
time.sleep(3)
get_resp(mcd_producer.delete("key", 0), 'Create mutation on producer')
time.sleep(10)
consumer.shutdown()
producer.shutdown()
mcd_consumer.shutdown()
mcd_producer.shutdown()
def setUp(self):
self.upr_client = UprClient(self.host, self.port)
self.mcd_client = McdClient(self.host, self.port)
if self.backend == RemoteServer.MCD:
resp = self.mcd_client.flush().next_response()
assert resp["status"] == SUCCESS, "Flush all is not enabled"
class UprTestCase(ParametrizedTestCase):
def setUp(self):
self.upr_client = UprClient(self.host, self.port)
self.mcd_client = McdClient(self.host, self.port)
if self.backend == RemoteServer.MCD:
resp = self.mcd_client.flush().next_response()
assert resp["status"] == SUCCESS, "Flush all is not enabled"
def tearDown(self):
self.upr_client.shutdown()
self.mcd_client.shutdown()
"""Basic upr open consumer connection test
Verifies that when the open upr consumer command is used there is a
connection instance that is created on the server and that when the
tcp connection is closed the connection is remove from the server"""
def test_open_consumer_connection_command(self):
op = self.upr_client.open_consumer("mystream")
response = op.next_response()
assert response["status"] == SUCCESS
op = self.mcd_client.stats("tap")
response = op.next_response()
assert response["value"]["mystream:type"] == "consumer"
self.upr_client.shutdown()
op = self.mcd_client.stats("tap")
response = op.next_response()
assert "mystream:type" not in response["value"]
"""Basic upr open producer connection test
Verifies that when the open upr producer command is used there is a
connection instance that is created on the server and that when the
tcp connection is closed the connection is remove from the server"""
def test_open_producer_connection_command(self):
op = self.upr_client.open_producer("mystream")
response = op.next_response()
assert response["status"] == SUCCESS
op = self.mcd_client.stats("tap")
response = op.next_response()
assert response["value"]["mystream:type"] == "producer"
self.upr_client.shutdown()
op = self.mcd_client.stats("tap")
response = op.next_response()
assert "mystream:type" not in response["value"]
"""Basic add stream test
This test verifies a simple add stream command. It expects that a stream
request message will be sent to the producer before a response for the
add stream command is returned."""
def test_add_stream_command(self):
op = self.upr_client.open_consumer("mystream")
response = op.next_response()
assert response["status"] == SUCCESS
op = self.upr_client.add_stream(0, 0)
response = op.next_response()
assert response["status"] == SUCCESS
"""Add stream command with no consumer vbucket
Attempts to add a stream when no vbucket exists on the consumer. The
client shoudl expect a not my vbucket response immediately"""
def test_add_stream_not_my_vbucket(self):
op = self.upr_client.open_consumer("mystream")
response = op.next_response()
assert response["status"] == SUCCESS
op = self.upr_client.add_stream(1025, 0)
response = op.next_response()
assert response["status"] == ERR_NOT_MY_VBUCKET
"""Add stream when stream exists
Creates a stream and then attempts to create another stream for the
same vbucket. Expects to fail with an exists error."""
def test_add_stream_exists(self):
op = self.upr_client.open_consumer("mystream")
response = op.next_response()
assert response["status"] == SUCCESS
op = self.upr_client.add_stream(0, 0)
response = op.next_response()
assert response["status"] == SUCCESS
op = self.upr_client.add_stream(0, 0)
response = op.next_response()
assert response["status"] == ERR_KEY_EEXISTS
def test_close_stream_command(self):
op = self.upr_client.close_stream(0)
response = op.next_response()
assert response["status"] == ERR_NOT_SUPPORTED
def test_get_failover_log_command(self):
op = self.upr_client.get_failover_log(0)
response = op.next_response()
assert response["status"] == ERR_NOT_SUPPORTED
"""Basic upr stream request
Opens a producer connection and sends a stream request command for
vbucket 0. Since no items exist in the server we should accept the
stream request and then send back a stream end message."""
def test_stream_request_command(self):
op = self.upr_client.open_producer("mystream")
response = op.next_response()
assert response["status"] == SUCCESS
op = self.upr_client.stream_req(0, 0, 0, 0, 0, 0)
while op.has_response():
response = op.next_response()
assert response["status"] == SUCCESS
@skipUnlessMcd
def test_stream_request_with_ops(self):
for i in range(10):
op = self.mcd_client.set("key" + str(i), "value", 0, 0, 0)
resp = op.next_response()
assert resp["status"] == SUCCESS
op = self.mcd_client.stats("vbucket-seqno")
resp = op.next_response()
assert resp["status"] == SUCCESS
end_seqno = int(resp["value"]["vb_0"])
op = self.upr_client.open_producer("mystream")
response = op.next_response()
assert response["status"] == SUCCESS
op = self.upr_client.stream_req(0, 0, 0, end_seqno, 0, 0)
while op.has_response():
response = op.next_response()
assert response["status"] == SUCCESS
print response
@skipUnlessMcd
def test_stream_request_with_deletes(self):
for i in range(10):
op = self.mcd_client.set("key" + str(i), "value", 0, 0, 0)
resp = op.next_response()
assert resp["status"] == SUCCESS
op = self.mcd_client.stats("vbucket-seqno")
resp = op.next_response()
assert resp["status"] == SUCCESS
end_seqno = int(resp["value"]["vb_0"])
op = self.upr_client.open_producer("mystream")
response = op.next_response()
assert response["status"] == SUCCESS
op = self.upr_client.stream_req(0, 0, 0, end_seqno, 0, 0)
while op.has_response():
response = op.next_response()
assert response["status"] == SUCCESS
print response