def test_streaming_zero_results(self):
try:
master_conn = get_connection()
master_conn.begin()
master_conn._execute(
"delete from vt_insert_test", {},
KEYSPACE_NAME,
'master',
keyranges=[keyrange.KeyRange(shard_names[self.shard_index])])
master_conn.commit()
# After deletion, should result zero.
stream_cursor = vtgate_cursor.StreamVTGateCursor(
master_conn,
KEYSPACE_NAME,
'master',
keyranges=[keyrange.KeyRange(shard_names[self.shard_index])])
stream_cursor.execute("select * from vt_insert_test", {})
rows = stream_cursor.fetchall()
rowcount = 0
for r in rows:
rowcount += 1
self.assertEqual(rowcount, 0)
except Exception, e:
self.fail("Failed with error %s %s" %
(str(e), traceback.print_exc()))
def setUp(self):
"""Updates schema, adding a table and populating it with data."""
super(BackupTest, self).setUp()
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION'] = '2'
self.table_name = 'vt_insert_test'
self.env.delete_table(self.table_name)
self.env.create_table(self.table_name, validate_deadline_s=120)
for keyspace, num_shards in zip(self.env.keyspaces, self.env.num_shards):
for shard_name in sharding_utils.get_shard_names(num_shards):
logging.info('Inserting %d rows into %s',
self.num_inserts, self.table_name)
kr = keyrange.KeyRange('') if num_shards == 1 else keyrange.KeyRange(
shard_name)
master_tablet = self.env.get_current_master_name(keyspace, shard_name)
master_cell = self.env.get_tablet_cell(master_tablet)
conn = self.env.get_vtgate_conn(master_cell)
cursor = conn.cursor(tablet_type='master', keyspace=keyspace,
keyranges=[kr], writable=True)
for i in xrange(self.num_inserts):
cursor.begin()
cursor.execute(
'insert into %s (msg, keyspace_id) values (:msg, :keyspace_id)' %
self.table_name, {'msg': 'test %d' % i, 'keyspace_id': 0})
cursor.commit()
cursor.close()
logging.info('Data insertion complete')
class TestFailures(unittest.TestCase):
def setUp(self):
self.shard_index = 0
self.master_tablet = shard_0_master
self.replica_tablet = shard_0_replica
def test_tablet_restart_read(self):
try:
replica_conn = get_connection()
except Exception, e:
self.fail("Connection to vtgate failed with error %s" % (str(e)))
self.replica_tablet.kill_vttablet()
with self.assertRaises(dbexceptions.DatabaseError):
replica_conn._execute(
"select 1 from vt_insert_test", {},
KEYSPACE_NAME,
'replica',
keyranges=[keyrange.KeyRange(shard_names[self.shard_index])])
proc = self.replica_tablet.start_vttablet()
try:
results = replica_conn._execute(
"select 1 from vt_insert_test", {},
KEYSPACE_NAME,
'replica',
keyranges=[keyrange.KeyRange(shard_names[self.shard_index])])
except Exception, e:
self.fail(
"Communication with shard %s replica failed with error %s" %
(shard_names[self.shard_index], str(e)))
def test_writes(self):
try:
master_conn = get_connection()
count = 10
master_conn.begin()
master_conn._execute(
"delete from vt_insert_test", {},
KEYSPACE_NAME,
'master',
keyranges=[keyrange.KeyRange(shard_names[self.shard_index])])
kid_list = shard_kid_map[shard_names[self.shard_index]]
for x in xrange(count):
keyspace_id = kid_list[count % len(kid_list)]
master_conn._execute(
"insert into vt_insert_test (msg, keyspace_id) values (%(msg)s, %(keyspace_id)s)",
{
'msg': 'test %s' % x,
'keyspace_id': keyspace_id
},
KEYSPACE_NAME,
'master',
keyspace_ids=[str(keyspace_id)])
master_conn.commit()
results, rowcount = master_conn._execute(
"select * from vt_insert_test", {},
KEYSPACE_NAME,
'master',
keyranges=[keyrange.KeyRange(shard_names[self.shard_index])
])[:2]
self.assertEqual(rowcount, count, "master fetch works")
except Exception, e:
logging.debug("Write failed with error %s" % str(e))
raise
def get_keyrange(shard_name):
kr = None
if shard_name == keyrange_constants.SHARD_ZERO:
kr = keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE)
else:
kr = keyrange.KeyRange(shard_name)
return kr
def get_keyrange_from_shard_name(keyspace, shard_name, db_type):
kr = None
if not is_sharded_keyspace(keyspace, db_type):
if shard_name == keyrange_constants.SHARD_ZERO:
kr = keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE)
else:
raise dbexceptions.DatabaseError(
'Invalid shard_name %s for keyspace %s', shard_name, keyspace)
else:
kr = keyrange.KeyRange(shard_name)
return kr
def test_batch_write(self):
try:
master_conn = get_connection()
count = 10
query_list = []
bind_vars_list = []
query_list.append("delete from vt_insert_test")
bind_vars_list.append({})
kid_list = shard_kid_map[shard_names[self.shard_index]]
for x in xrange(count):
keyspace_id = kid_list[count % len(kid_list)]
query_list.append(
"insert into vt_insert_test (msg, keyspace_id) values (%(msg)s, %(keyspace_id)s)"
)
bind_vars_list.append({
'msg': 'test %s' % x,
'keyspace_id': keyspace_id
})
query_list.append("delete from vt_a")
bind_vars_list.append({})
for x in xrange(count):
keyspace_id = kid_list[count % len(kid_list)]
query_list.append(
"insert into vt_a (eid, id, keyspace_id) values (%(eid)s, %(id)s, %(keyspace_id)s)"
)
bind_vars_list.append({
'eid': x,
'id': x,
'keyspace_id': keyspace_id
})
master_conn.begin()
master_conn._execute_batch(
query_list,
bind_vars_list,
KEYSPACE_NAME,
'master',
keyspace_ids=[str(kid) for kid in kid_list])
master_conn.commit()
results, rowcount, _, _ = master_conn._execute(
"select * from vt_insert_test", {},
KEYSPACE_NAME,
'master',
keyranges=[keyrange.KeyRange(shard_names[self.shard_index])])
self.assertEqual(rowcount, count)
results, rowcount, _, _ = master_conn._execute(
"select * from vt_a", {},
KEYSPACE_NAME,
'master',
keyranges=[keyrange.KeyRange(shard_names[self.shard_index])])
self.assertEqual(rowcount, count)
except Exception, e:
self.fail("Write failed with error %s" % str(e))
def test_batch_read(self):
try:
master_conn = get_connection()
count = 10
master_conn.begin()
master_conn._execute(
"delete from vt_insert_test", {},
KEYSPACE_NAME,
'master',
keyranges=[keyrange.KeyRange(shard_names[self.shard_index])])
kid_list = shard_kid_map[shard_names[self.shard_index]]
for x in xrange(count):
keyspace_id = kid_list[count % len(kid_list)]
master_conn._execute(
"insert into vt_insert_test (msg, keyspace_id) values (%(msg)s, %(keyspace_id)s)",
{
'msg': 'test %s' % x,
'keyspace_id': keyspace_id
},
KEYSPACE_NAME,
'master',
keyspace_ids=[str(keyspace_id)])
master_conn.commit()
master_conn.begin()
master_conn._execute(
"delete from vt_a", {},
KEYSPACE_NAME,
'master',
keyranges=[keyrange.KeyRange(shard_names[self.shard_index])])
for x in xrange(count):
keyspace_id = kid_list[count % len(kid_list)]
master_conn._execute("insert into vt_a (eid, id, keyspace_id) \
values (%(eid)s, %(id)s, %(keyspace_id)s)", {
'eid': x,
'id': x,
'keyspace_id': keyspace_id
},
KEYSPACE_NAME,
'master',
keyspace_ids=[str(keyspace_id)])
master_conn.commit()
rowsets = master_conn._execute_batch(
["select * from vt_insert_test", "select * from vt_a"],
[{}, {}],
KEYSPACE_NAME,
'master',
keyspace_ids=[str(kid) for kid in kid_list])
self.assertEqual(rowsets[0][1], count)
self.assertEqual(rowsets[1][1], count)
except Exception, e:
self.fail("Write failed with error %s %s" %
(str(e), traceback.print_exc()))
def exec_query(conn, title, query, response, keyspace=None, kr=None): # pylint: disable=missing-docstring
if kr:
# v2 cursor to address individual shards directly, for debug display
cursor = conn.cursor(tablet_type="master",
keyspace=keyspace,
keyranges=[keyrange.KeyRange(kr)])
else:
# v3 cursor is automated
cursor = conn.cursor(tablet_type="master",
keyspace=keyspace,
writable=True)
try:
if not query or query == "undefined":
return
if query.startswith("select"):
cursor.execute(query, {})
else:
cursor.begin()
cursor.execute(query, {})
cursor.commit()
response[title] = {
"title": title,
"description": cursor.description,
"rowcount": cursor.rowcount,
"lastrowid": cursor.lastrowid,
"results": cursor.results,
}
cursor.close()
except Exception as e: # pylint: disable=broad-except
response[title] = {
"title": title,
"error": str(e),
}
def test_execute_entity_ids(self):
try:
master_conn = get_connection()
count = 10
master_conn.begin()
master_conn._execute(
"delete from vt_a", {},
KEYSPACE_NAME,
'master',
keyranges=[keyrange.KeyRange(shard_names[self.shard_index])])
eid_map = {}
kid_list = shard_kid_map[shard_names[self.shard_index]]
for x in xrange(count):
keyspace_id = kid_list[count % len(kid_list)]
eid_map[x] = str(keyspace_id)
master_conn._execute("insert into vt_a (eid, id, keyspace_id) \
values (%(eid)s, %(id)s, %(keyspace_id)s)", {
'eid': x,
'id': x,
'keyspace_id': keyspace_id
},
KEYSPACE_NAME,
'master',
keyspace_ids=[str(keyspace_id)])
master_conn.commit()
results, rowcount, _, _ = master_conn._execute_entity_ids(
"select * from vt_a", {}, KEYSPACE_NAME, 'master', eid_map,
'id')
self.assertEqual(rowcount, count, "entity_ids works")
except Exception, e:
self.fail("Execute entity ids failed with error %s %s" %
(str(e), traceback.print_exc()))
def _insert_values(self,
table,
count,
db_type='master',
keyspace='source_keyspace'):
result = self.insert_index
conn = self._vtdb_conn()
cursor = conn.cursor(None,
conn,
keyspace,
db_type,
keyranges=[
keyrange.KeyRange(
keyrange_constants.NON_PARTIAL_KEYRANGE)
],
writable=True)
for i in xrange(count):
conn.begin()
cursor.execute(
"insert into %s (id, msg) values(%u, 'value %u')" %
(table, self.insert_index, self.insert_index), {})
conn.commit()
self.insert_index += 1
conn.close()
return result
def _check_client_conn_redirection(self,
source_ks,
destination_ks,
db_types,
servedfrom_db_types,
moved_tables=None):
# check that the ServedFrom indirection worked correctly.
if moved_tables is None:
moved_tables = []
conn = self._vtdb_conn()
for db_type in servedfrom_db_types:
for tbl in moved_tables:
try:
rows = conn._execute(
"select * from %s" % tbl, {},
destination_ks,
db_type,
keyranges=[
keyrange.KeyRange(
keyrange_constants.NON_PARTIAL_KEYRANGE)
])
logging.debug("Select on %s.%s returned %d rows" %
(db_type, tbl, len(rows)))
except Exception, e:
self.fail("Execute failed w/ exception %s" % str(e))
def test_keyrange_correctness(self):
kr = keyrange.KeyRange('')
self.assertEqual(kr.Start, keyrange_constants.MIN_KEY)
self.assertEqual(kr.End, keyrange_constants.MAX_KEY)
self.assertEqual(str(kr), keyrange_constants.NON_PARTIAL_KEYRANGE)
kr = keyrange.KeyRange('-')
self.assertEqual(kr.Start, keyrange_constants.MIN_KEY)
self.assertEqual(kr.End, keyrange_constants.MAX_KEY)
self.assertEqual(str(kr), keyrange_constants.NON_PARTIAL_KEYRANGE)
for kr_str in int_shard_kid_map:
Start_raw, End_raw = kr_str.split('-')
kr = keyrange.KeyRange(kr_str)
self.assertEqual(kr.Start, Start_raw.strip().decode('hex'))
self.assertEqual(kr.End, End_raw.strip().decode('hex'))
self.assertEqual(str(kr), kr_str)
def create_shard_routing(class_, *pargs, **kargs):
"""This creates the ShardRouting object based on the kargs.
This prunes the routing kargs so as not to interfere with the
actual database method.
Args:
*pargs: Positional arguments
**kargs: Routing key-value params. These are used to determine routing.
There are two mutually exclusive mechanisms to indicate routing.
1. entity_id_map {"entity_id_column": entity_id_value} where entity_id_column
could be the sharding key or a lookup based entity column of this table. This
helps determine the keyspace_ids for the cursor.
2. keyrange - This helps determine the keyrange for the cursor.
Returns:
ShardRouting object and modified kargs
"""
lookup_cursor_method = pargs[0]
routing = db_object.ShardRouting(class_.keyspace)
entity_id_map = None
entity_id_map = kargs.get("entity_id_map", None)
if entity_id_map is None:
kr = None
key_range = kargs.get("keyrange", None)
if isinstance(key_range, keyrange.KeyRange):
kr = key_range
else:
kr = keyrange.KeyRange(key_range)
if kr is not None:
routing.keyrange = kr
# Both entity_id_map and keyrange have been evaluated. Return.
return routing
# entity_id_map is not None
if len(entity_id_map) != 1:
dbexceptions.ProgrammingError("Invalid entity_id_map '%s'" %
entity_id_map)
entity_id_col = entity_id_map.keys()[0]
entity_id = entity_id_map[entity_id_col]
#TODO: the current api means that if a table doesn't have the sharding key column name
# then it cannot pass in sharding key for routing purposes. Will this cause
# extra load on lookup db/cache ? This is cleaner from a design perspective.
if entity_id_col == class_.sharding_key_column_name:
# Routing using sharding key.
routing.sharding_key = entity_id
if not class_.is_sharding_key_valid(routing.sharding_key):
raise dbexceptions.InternalError("Invalid sharding_key %s" %
routing.sharding_key)
else:
# Routing using lookup based entity.
routing.entity_column_name = entity_id_col
routing.entity_id_sharding_key_map = class_.lookup_sharding_key_from_entity_id(
lookup_cursor_method, entity_id_col, entity_id)
return routing
def _delete_all(table_name):
vtgate_conn = get_connection()
# This write is to set up the test with fresh insert
# and hence performing it directly on the connection.
vtgate_conn.begin()
vtgate_conn._execute('delete from %s' % table_name, {},
tablet_type='master',
keyspace_name=KEYSPACE_NAME,
keyranges=[keyrange.KeyRange('')])
vtgate_conn.commit()
def test_retry_txn_pool_full(self):
master_conn = get_connection()
master_conn._execute(
"set vt_transaction_cap=1", {},
KEYSPACE_NAME, 'master',
keyranges=[keyrange.KeyRange(shard_names[self.shard_index])])
master_conn.begin()
master_conn2 = get_connection()
master_conn2.begin()
master_conn.commit()
master_conn._execute(
"set vt_transaction_cap=20", {},
KEYSPACE_NAME, 'master',
keyranges=[keyrange.KeyRange(shard_names[self.shard_index])])
master_conn.begin()
master_conn._execute(
"delete from vt_insert_test", {},
KEYSPACE_NAME, 'master',
keyranges=[keyrange.KeyRange(shard_names[self.shard_index])])
master_conn.commit()
def get_keyrange(shard_name):
kr = None
if shard_name == keyrange_constants.SHARD_ZERO:
kr = keyrange_constants.NON_PARTIAL_KEYRANGE
else:
kr_parts = shard_name.split('-')
if len(kr_parts) != 2:
raise dbexceptions.DatabaseError(
'Invalid shard_name %s for keyspace %s', shard_name, keyspace)
kr = keyrange.KeyRange(
(kr_parts[0].decode('hex'), kr_parts[1].decode('hex')))
return kr
def read_rows(self, keyspace, num_reads, num_rows, cell,
tablet_type='replica'):
logging.info('Reading %d rows from %s', num_reads, self.table_name)
conn = self.env.get_vtgate_conn(cell)
cursor = conn.cursor(
tablet_type=tablet_type, keyspace=keyspace,
keyranges=[keyrange.KeyRange('')])
for i in [random.randint(0, num_rows - 1) for _ in xrange(num_reads)]:
cursor.execute(
'select * from %s where msg = "test %d"' % (self.table_name, i), {})
cursor.close()
logging.info('Data reads complete')
def get_keyrange_from_shard_name(keyspace, shard_name):
kr = None
# db_type is immaterial here.
if not is_sharded_keyspace(keyspace, 'replica'):
if shard_name == keyrange_constants.SHARD_ZERO:
kr = keyrange_constants.NON_PARTIAL_KEYRANGE
else:
raise dbexceptions.DatabaseError('Invalid shard_name %s for keyspace %s', shard_name, keyspace)
else:
kr_parts = shard_name.split('-')
if len(kr_parts) != 2:
raise dbexceptions.DatabaseError('Invalid shard_name %s for keyspace %s', shard_name, keyspace)
kr = keyrange.KeyRange((kr_parts[0].decode('hex'), kr_parts[1].decode('hex')))
return kr
def insert_rows(self, keyspace, num_rows, starting_index=0):
logging.info('Inserting %d rows into %s', num_rows, self.table_name)
master_cell = self.env.get_current_master_cell(keyspace)
conn = self.env.get_vtgate_conn(master_cell)
cursor = conn.cursor(tablet_type='master', keyspace=keyspace,
keyranges=[keyrange.KeyRange('')], writable=True)
for i in xrange(starting_index, starting_index + num_rows):
cursor.begin()
cursor.execute(
'insert into %s (msg, keyspace_id) values (:msg, :keyspace_id)' %
self.table_name, {'msg': 'test %d' % i, 'keyspace_id': 0})
cursor.commit()
cursor.close()
logging.info('Data insertion complete')
def do_write(count, shard_index):
kid_list = shard_kid_map[shard_names[shard_index]]
master_conn = get_connection()
master_conn.begin()
master_conn._execute(
"delete from vt_insert_test", {},
KEYSPACE_NAME, 'master',
keyranges=[keyrange.KeyRange(shard_names[shard_index])])
for x in xrange(count):
keyspace_id = kid_list[count%len(kid_list)]
master_conn._execute(
"insert into vt_insert_test (msg, keyspace_id) values (%(msg)s, %(keyspace_id)s)",
{'msg': 'test %s' % x, 'keyspace_id': keyspace_id},
KEYSPACE_NAME, 'master', keyspace_ids=[str(keyspace_id)])
master_conn.commit()
def test_streaming_fetchone(self):
try:
count = 100
do_write(count, self.shard_index)
# Fetch one.
master_conn = get_connection()
stream_cursor = vtgate_cursor.StreamVTGateCursor(
master_conn,
KEYSPACE_NAME, 'master',
keyranges=[keyrange.KeyRange(shard_names[self.shard_index])])
stream_cursor.execute("select * from vt_insert_test", {})
rows = stream_cursor.fetchone()
self.assertTrue(type(rows) == tuple, "Received a valid row")
stream_cursor.close()
except Exception, e:
self.fail("Failed with error %s %s" % (str(e), traceback.print_exc()))
def test_interleaving(self):
tablet_type = "master"
try:
vtgate_conn = get_connection()
vtgate_conn.begin()
vtgate_conn._execute(
"delete from vt_insert_test", {},
KEYSPACE_NAME, tablet_type,
keyranges=[keyrange.KeyRange(shard_names[self.shard_index])])
kid_list = shard_kid_map[shard_names[self.shard_index]]
count = len(kid_list)
for x in xrange(count):
keyspace_id = kid_list[x]
vtgate_conn._execute(
"insert into vt_insert_test (msg, keyspace_id) values (%(msg)s, %(keyspace_id)s)",
{'msg': 'test %s' % x, 'keyspace_id': keyspace_id},
KEYSPACE_NAME, tablet_type, keyspace_ids=[str(keyspace_id)])
vtgate_conn.commit()
vtgate_conn2 = get_connection()
query = "select keyspace_id from vt_insert_test where keyspace_id = %(kid)s"
thd = threading.Thread(target=self._query_lots, args=(
vtgate_conn2,
query,
{'kid': kid_list[0]},
KEYSPACE_NAME,
tablet_type,
[str(kid_list[0])]))
thd.start()
for i in xrange(count):
(result, _, _, _) = vtgate_conn._execute(query,
{'kid':kid_list[i]},
KEYSPACE_NAME, tablet_type,
keyspace_ids=[str(kid_list[i])])
self.assertEqual(result, [(kid_list[i],)])
if i % 10 == 0:
vtgate_conn._stream_execute(query, {'kid':kid_list[i]}, KEYSPACE_NAME,
tablet_type,
keyspace_ids=[str(kid_list[i])])
while 1:
result = vtgate_conn._stream_next()
if not result:
break
self.assertEqual(result, (kid_list[i],))
thd.join()
except Exception, e:
self.fail("Failed with error %s %s" % (str(e), traceback.print_exc()))
def setUpModule():
logging.debug('in setUpModule')
try:
environment.topo_server().setup()
# start mysql instance external to the test
setup_procs = [shard_0_master.init_mysql(),
shard_0_replica1.init_mysql(),
]
utils.wait_procs(setup_procs)
setup_tablets()
setup_vtgate()
# After VTGate comes up, populate it with some initial data
initial_writes(0, keyrange.KeyRange(''))
except Exception, e:
logging.exception('error during set up: %s', e)
tearDownModule()
raise
def test_streaming_fetchall(self):
try:
count = 100
do_write(count, self.shard_index)
# Fetch all.
master_conn = get_connection()
stream_cursor = vtgate_cursor.StreamVTGateCursor(
master_conn,
KEYSPACE_NAME, 'master',
keyranges=[keyrange.KeyRange(shard_names[self.shard_index])])
stream_cursor.execute("select * from vt_insert_test", {})
rows = stream_cursor.fetchall()
rowcount = 0
for r in rows:
rowcount +=1
self.assertEqual(rowcount, count)
stream_cursor.close()
except Exception, e:
self.fail("Failed with error %s %s" % (str(e), traceback.print_exc()))
def _check_client_conn_redirection(
self, destination_ks, servedfrom_db_types,
moved_tables=None):
# check that the ServedFrom indirection worked correctly.
if moved_tables is None:
moved_tables = []
conn = self._vtdb_conn()
for db_type in servedfrom_db_types:
for tbl in moved_tables:
try:
rows = conn._execute(
'select * from %s' % tbl, {}, tablet_type=db_type,
keyspace_name=destination_ks,
keyranges=[keyrange.KeyRange(
keyrange_constants.NON_PARTIAL_KEYRANGE)])
logging.debug(
'Select on %s.%s returned %d rows', db_type, tbl, len(rows))
except Exception, e: # pylint: disable=broad-except
self.fail('Execute failed w/ exception %s' % str(e))
def compute_kr_list(self):
"""compute the keyrange list for parallel queries.
"""
kr_chunks = []
# we only support 256 shards for now
min_key_hex = int('00', base=16)
max_key_hex = int('100', base=16)
kr = min_key_hex
span = (max_key_hex - min_key_hex) / self.num_tasks
kr_chunks.append('')
for i in xrange(self.num_tasks):
kr += span
kr_chunks.append('%x' % kr)
kr_chunks[-1] = ''
for i in xrange(len(kr_chunks) - 1):
kr_str = str(keyrange.KeyRange((
kr_chunks[i],
kr_chunks[i + 1],
)))
self.keyrange_list.append(kr_str)
def test_streaming_multishards(self):
try:
count = 100
do_write(count, 0)
do_write(count, 1)
vtgate_conn = get_connection()
stream_cursor = vtgate_conn.cursor(
KEYSPACE_NAME,
'master',
keyranges=[
keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE)
],
cursorclass=vtgate_cursor.StreamVTGateCursor)
stream_cursor.execute("select * from vt_insert_test", {})
rows = stream_cursor.fetchall()
rowcount = 0
for row in rows:
rowcount += 1
self.assertEqual(rowcount, count * 2)
stream_cursor.close()
except Exception, e:
self.fail("Failed with error %s %s" %
(str(e), traceback.print_exc()))
class TestMasterBuffering(BaseTestCase):
shard_index = 0
keyrange = keyrange.KeyRange('')
def setUp(self):
super(TestMasterBuffering, self).setUp()
restart_vtgate(extra_args=[
'-enable_fake_master_buffer',
'-buffer_keyspace',
KEYSPACE_NAME,
'-buffer_shard',
SHARD_NAMES[self.shard_index],
'-fake_buffer_delay',
'1ms',
])
def get_sucessful_buffered_requests(self):
return utils.vtgate.get_vars()['BufferedRequestsSuccessful']
def test_tx_is_buffered(self):
"""Tests that for a transaction, we buffer exactly one request."""
vtgate_conn = get_connection()
kid_list = SHARD_KID_MAP[SHARD_NAMES[self.shard_index]]
keyspace_id = kid_list[0]
initial_buffered = self.get_sucessful_buffered_requests()
cursor = vtgate_conn.cursor(tablet_type='master',
keyspace=KEYSPACE_NAME,
keyspace_ids=[pack_kid(keyspace_id)],
writable=True)
cursor.begin()
cursor.execute(
'insert into vt_insert_test (msg, keyspace_id) '
'values (:msg, :keyspace_id)', {
'msg': 'test %s' % 1000,
'keyspace_id': keyspace_id
})
cursor.execute('select * from vt_insert_test', {})
cursor.rollback()
num_buffered = self.get_sucessful_buffered_requests(
) - initial_buffered
# No matter how many requests there were in the transaction, we should only
# buffer one request (the Begin to the vttablet).
self.assertEqual(num_buffered, 1)
def test_master_read_is_buffered(self):
"""Tests that we buffer master reads."""
vtgate_conn = get_connection()
kid_list = SHARD_KID_MAP[SHARD_NAMES[self.shard_index]]
keyspace_id = kid_list[0]
initial_buffered = self.get_sucessful_buffered_requests()
cursor = vtgate_conn.cursor(tablet_type='master',
keyspace=KEYSPACE_NAME,
keyspace_ids=[pack_kid(keyspace_id)])
cursor.execute('select * from vt_insert_test', {})
num_buffered = self.get_sucessful_buffered_requests(
) - initial_buffered
self.assertEqual(num_buffered, 1)
def test_replica_read_is_not_buffered(self):
"""Tests that we do not buffer replica reads."""
vtgate_conn = get_connection()
initial_buffered = self.get_sucessful_buffered_requests()
vtgate_conn._execute('select * from vt_insert_test', {},
tablet_type='replica',
keyspace_name=KEYSPACE_NAME,
keyranges=[self.keyrange])
num_buffered = self.get_sucessful_buffered_requests(
) - initial_buffered
self.assertEqual(num_buffered, 0)
"Communication with shard %s replica failed with error %s" %
(shard_names[self.shard_index], str(e)))
def test_vtgate_restart_read(self):
global vtgate_server, vtgate_port
try:
replica_conn = get_connection()
except Exception, e:
self.fail("Connection to vtgate failed with error %s" % (str(e)))
utils.vtgate_kill(vtgate_server)
with self.assertRaises(dbexceptions.OperationalError):
replica_conn._execute(
"select 1 from vt_insert_test", {},
KEYSPACE_NAME,
'replica',
keyranges=[keyrange.KeyRange(shard_names[self.shard_index])])
vtgate_server, vtgate_port = utils.vtgate_start(vtgate_port)
replica_conn = get_connection()
try:
results = replica_conn._execute(
"select 1 from vt_insert_test", {},
KEYSPACE_NAME,
'replica',
keyranges=[keyrange.KeyRange(shard_names[self.shard_index])])
except Exception, e:
self.fail(
"Communication with shard %s replica failed with error %s" %
(shard_names[self.shard_index], str(e)))
def test_tablet_restart_stream_execute(self):
try: