def test_database_filter(self):
start_position = _get_master_current_position()
master_tablet.mquery('other_database', _create_vt_insert_test)
self._exec_vt_txn(self._populate_vt_insert_test)
logging.debug('test_database_filter: starting @ %s', start_position)
master_conn = self._get_vtgate_stream_conn()
for event, _ in master_conn.update_stream(
'test_keyspace', topodata_pb2.MASTER,
event=query_pb2.EventToken(shard='0', position=start_position),
shard='0'):
for statement in event.statements:
self.assertNotEqual(statement.category, 2, # query_pb2.StreamEvent.DDL
"query using other_database wasn't filtered out")
break
master_conn.close()
def test_update_stream_interrupt(self):
"""Checks that a running query is terminated on going non-serving."""
# Make sure the replica is replica type.
utils.run_vtctl(
['ChangeSlaveType', replica_tablet.tablet_alias, 'replica'])
logging.debug('sleeping a bit for the replica action to complete')
utils.wait_for_tablet_type(replica_tablet.tablet_alias, 'replica', 30)
# Save current position, insert some data.
start_position = _get_repl_current_position()
logging.debug('test_update_stream_interrupt starting @ %s',
start_position)
self._exec_vt_txn(self._populate_vt_a(1))
self._exec_vt_txn(['delete from vt_a'])
# Start an Update Stream from the slave. When we get the data, go to spare.
# That should interrupt the streaming RPC.
replica_conn = self._get_vtgate_stream_conn()
first = True
txn_count = 0
try:
for event, resume_timestamp in replica_conn.update_stream(
'test_keyspace',
topodata_pb2.REPLICA,
event=query_pb2.EventToken(shard='0',
position=start_position),
shard='0'):
logging.debug('test_update_stream_interrupt got event(%d): %s',
resume_timestamp, event)
if first:
utils.run_vtctl([
'ChangeSlaveType', replica_tablet.tablet_alias, 'spare'
])
utils.wait_for_tablet_type(replica_tablet.tablet_alias,
'spare', 30)
first = False
else:
if event.event_token.position:
txn_count += 1
self.assertFail('update_stream terminated with no exception')
except dbexceptions.DatabaseError as e:
self.assertIn('context canceled', str(e))
self.assertFalse(first)
logging.debug('Streamed %d transactions before exiting', txn_count)
replica_conn.close()
def test_ddl(self):
"""Asks for all statements since we started, find the DDL."""
start_position = master_start_position
logging.debug('test_ddl: starting @ %s', start_position)
master_conn = self._get_vtgate_stream_conn()
found = False
for event, _ in master_conn.update_stream(
'test_keyspace', topodata_pb2.MASTER,
event=query_pb2.EventToken(shard='0', position=start_position),
shard='0'):
for statement in event.statements:
if statement.sql == _create_vt_insert_test:
found = True
break
break
master_conn.close()
self.assertTrue(found, "didn't get right sql")
def test_service_switch(self):
"""tests the service switch from disable -> enable -> disable."""
# make the replica spare
utils.run_vtctl(['ChangeSlaveType', replica_tablet.tablet_alias, 'spare'])
utils.wait_for_tablet_type(replica_tablet.tablet_alias, 'spare')
# Check UpdateStreamState is disabled.
v = utils.get_vars(replica_tablet.port)
if v['UpdateStreamState'] != 'Disabled':
self.fail("Update stream service should be 'Disabled' but is '%s'" %
v['UpdateStreamState'])
start_position = _get_repl_current_position()
# Make sure we can't start a new request to vttablet directly.
_, stderr = utils.run_vtctl(['VtTabletUpdateStream',
'-position', start_position,
replica_tablet.tablet_alias],
expect_fail=True)
self.assertIn('operation not allowed in state NOT_SERVING', stderr)
# Make sure we can't start a new request through vtgate.
replica_conn = self._get_vtgate_stream_conn()
try:
for event, resume_timestamp in replica_conn.update_stream(
'test_keyspace', topodata_pb2.REPLICA,
event=query_pb2.EventToken(shard='0', position=start_position),
shard='0'):
self.assertFail('got event(%d): %s' % (resume_timestamp, str(event)))
self.assertFail('update_stream terminated with no exception')
except dbexceptions.DatabaseError as e:
self.assertIn(vtgate_gateway_flavor().no_tablet_found_message(), str(e))
# Go back to replica.
utils.run_vtctl(
['ChangeSlaveType', replica_tablet.tablet_alias, 'replica'])
utils.wait_for_tablet_type(replica_tablet.tablet_alias, 'replica')
# Check UpdateStreamState is enabled.
v = utils.get_vars(replica_tablet.port)
if v['UpdateStreamState'] != 'Enabled':
self.fail("Update stream service should be 'Enabled' but is '%s'" %
v['UpdateStreamState'])
def test_set_insert_id(self):
start_position = _get_master_current_position()
self._exec_vt_txn(
['SET INSERT_ID=1000000'] + self._populate_vt_insert_test)
logging.debug('test_set_insert_id: starting @ %s', start_position)
master_conn = self._get_vtgate_stream_conn()
expected_id = 1000000
for event, _ in master_conn.update_stream(
'test_keyspace', topodata_pb2.MASTER,
event=query_pb2.EventToken(shard='0', position=start_position),
shard='0'):
for statement in event.statements:
fields, rows = proto3_encoding.convert_stream_event_statement(statement)
self.assertEqual(fields[0], 'id')
self.assertEqual(rows[0][0], expected_id)
expected_id += 1
break
if expected_id != 1000004:
self.fail('did not get my four values!')
master_conn.close()
def test_stream_parity(self):
"""Tests parity of streams between master and replica for the same writes.
Also tests transactions are retrieved properly.
"""
timeout = 30
while True:
master_position = _get_master_current_position()
replica_position = _get_repl_current_position()
if master_position == replica_position:
break
timeout = utils.wait_step(
'%s == %s' % (master_position, replica_position), timeout)
logging.debug('run_test_stream_parity starting @ %s', master_position)
self._exec_vt_txn(self._populate_vt_a(15))
self._exec_vt_txn(self._populate_vt_b(14))
self._exec_vt_txn(['delete from vt_a'])
self._exec_vt_txn(['delete from vt_b'])
# get master events
master_conn = self._get_vtgate_stream_conn()
master_events = []
for event, resume_timestamp in master_conn.update_stream(
'test_keyspace',
topodata_pb2.MASTER,
event=query_pb2.EventToken(shard='0',
position=master_position),
shard='0'):
logging.debug('Got master event(%d): %s', resume_timestamp, event)
master_events.append(event)
if len(master_events) == 4:
break
master_conn.close()
# get replica events
replica_conn = self._get_vtgate_stream_conn()
replica_events = []
for event, resume_timestamp in replica_conn.update_stream(
'test_keyspace',
topodata_pb2.REPLICA,
event=query_pb2.EventToken(shard='0',
position=replica_position),
shard='0'):
logging.debug('Got slave event(%d): %s', resume_timestamp, event)
replica_events.append(event)
if len(replica_events) == 4:
break
replica_conn.close()
# and compare
if len(master_events) != len(replica_events):
logging.debug(
'Test Failed - # of records mismatch, master %s replica %s',
master_events, replica_events)
for master_event, replica_event in zip(master_events, replica_events):
# The timestamp is from when the event was written to the binlogs.
# the master uses the timestamp of when it wrote it originally,
# the slave of when it applied the logs. These can differ and make this
# test flaky. So we just blank them out, easier. We really want to
# compare the replication positions.
master_event.event_token.timestamp = 123
replica_event.event_token.timestamp = 123
self.assertEqual(
master_event, replica_event,
"Test failed, data mismatch - master '%s' and replica '%s'" %
(master_event, replica_event))
logging.debug('Test Writes: PASS')
def test_event_token_fresher(self):
"""event_token.fresher test suite."""
test_cases = [
{
'ev1': None,
'ev2': None,
'expected': -1,
},
{
'ev1': query_pb2.EventToken(timestamp=123, ),
'ev2': None,
'expected': -1,
},
{
'ev1': None,
'ev2': query_pb2.EventToken(timestamp=123, ),
'expected': -1,
},
{
'ev1': query_pb2.EventToken(timestamp=123, ),
'ev2': query_pb2.EventToken(timestamp=123, ),
'expected': -1,
},
{
'ev1': query_pb2.EventToken(timestamp=200, ),
'ev2': query_pb2.EventToken(timestamp=100, ),
'expected': 100,
},
{
'ev1': query_pb2.EventToken(timestamp=100, ),
'ev2': query_pb2.EventToken(timestamp=200, ),
'expected': -100,
},
{
# Test cases with not enough information to compare.
'ev1': query_pb2.EventToken(timestamp=100, ),
'ev2': query_pb2.EventToken(timestamp=100, ),
'expected': -1,
},
{
'ev1': query_pb2.EventToken(
timestamp=100,
shard='s1',
),
'ev2': query_pb2.EventToken(
timestamp=100,
shard='s2',
),
'expected': -1,
},
{
'ev1': query_pb2.EventToken(
timestamp=100,
shard='s1',
),
'ev2': query_pb2.EventToken(
timestamp=100,
shard='s1',
),
'expected': -1,
},
{
'ev1':
query_pb2.EventToken(
timestamp=100,
shard='s1',
position='pos1',
),
'ev2':
query_pb2.EventToken(
timestamp=100,
shard='s1',
),
'expected':
-1,
},
{
'ev1':
query_pb2.EventToken(
timestamp=100,
shard='s1',
),
'ev2':
query_pb2.EventToken(
timestamp=100,
shard='s1',
position='pos2',
),
'expected':
-1,
},
{
'ev1':
query_pb2.EventToken(
timestamp=100,
shard='s1',
position='pos1', # invalid on purpose
),
'ev2':
query_pb2.EventToken(
timestamp=100,
shard='s1',
position='pos2', # invalid on purpose
),
'expected':
-1,
},
{
'ev1':
query_pb2.EventToken(
timestamp=100,
shard='s1',
position='MariaDB/0-1-123', # valid but different
),
'ev2':
query_pb2.EventToken(
timestamp=100,
shard='s1',
position=
'MySQL56/33333333-3333-3333-3333-333333333333:456-789',
),
'expected':
-1,
},
{
# MariaDB test cases.
'ev1':
query_pb2.EventToken(
timestamp=100,
shard='s1',
position='MariaDB/0-1-200',
),
'ev2':
query_pb2.EventToken(
timestamp=100,
shard='s1',
position='MariaDB/0-1-100',
),
'expected':
100,
},
{
'ev1':
query_pb2.EventToken(
timestamp=100,
shard='s1',
position='MariaDB/0-1-100',
),
'ev2':
query_pb2.EventToken(
timestamp=100,
shard='s1',
position='MariaDB/0-1-200',
),
'expected':
-100,
},
{
'ev1':
query_pb2.EventToken(
timestamp=100,
shard='s1',
position='MariaDB/0-1-100',
),
'ev2':
query_pb2.EventToken(
timestamp=100,
shard='s1',
position='MariaDB/0-1-100',
),
'expected':
0,
},
{
# MySQL56 test cases, not supported yet.
'ev1':
query_pb2.EventToken(
timestamp=100,
shard='s1',
position=
'MySQL56/33333333-3333-3333-3333-333333333333:1-200',
),
'ev2':
query_pb2.EventToken(
timestamp=100,
shard='s1',
position=
'MySQL56/33333333-3333-3333-3333-333333333333:1-100',
),
'expected':
-1, # Should be: 1,
},
{
'ev1':
query_pb2.EventToken(
timestamp=100,
shard='s1',
position=
'MySQL56/33333333-3333-3333-3333-333333333333:1-100',
),
'ev2':
query_pb2.EventToken(
timestamp=100,
shard='s1',
position=
'MySQL56/33333333-3333-3333-3333-333333333333:1-200',
),
'expected':
-1,
},
{
'ev1':
query_pb2.EventToken(
timestamp=100,
shard='s1',
position=
'MySQL56/33333333-3333-3333-3333-333333333333:1-100',
),
'ev2':
query_pb2.EventToken(
timestamp=100,
shard='s1',
position=
'MySQL56/33333333-3333-3333-3333-333333333333:1-100',
),
'expected':
-1, # Should be: 0,
}
]
for tcase in test_cases:
got = event_token.fresher(tcase['ev1'], tcase['ev2'])
self.assertEqual(
got, tcase['expected'],
'got %d but expected %d for Fresher(%s, %s)' %
(got, tcase['expected'], tcase['ev1'], tcase['ev2']))
class TestEcho(TestPythonClientBase):
"""Send queries to the server, check the returned result matches."""
echo_prefix = 'echo://'
query = (
u'test query with bind variables: :int :float :bytes, unicode: '
u'\u6211\u80fd\u541e\u4e0b\u73bb\u7483\u800c\u4e0d\u50b7\u8eab\u9ad4'
).encode('utf-8')
query_echo = (
u'test query with bind variables: :int :float :bytes, unicode: '
u'\u6211\u80fd\u541e\u4e0b\u73bb\u7483\u800c\u4e0d\u50b7\u8eab\u9ad4'
).encode('utf-8')
keyspace = 'test_keyspace'
shards = ['-80', '80-']
shards_echo = '[-80 80-]'
keyspace_ids = ['\x01\x02\x03\x04', '\x05\x06\x07\x08']
keyspace_ids_echo = '[[1 2 3 4] [5 6 7 8]]'
# FIXME(alainjobart) using a map for the entities makes it impossible to
# guarantee the order of the entities in the query. It is really an API
# problem here? For this test, however, I'll just use a single value for now
entity_keyspace_ids = {
123: '\x01\x02\x03',
# 2.0: '\x04\x05\x06',
# '\x01\x02\x03': '\x07\x08\x09',
}
# entity_keyspace_ids_echo = ('[type:INT64 value:"123" '
# 'keyspace_id:"\\001\\002\\003" '
# 'type:FLOAT64 value:"2" '
# 'keyspace_id:"\\004\\005\\006" '
# 'type:VARBINARY value:"\\001\\002\\003" '
# 'keyspace_id:"\\007\\010\\t" ]')
entity_keyspace_ids_echo = ('[type:INT64 value:"123" '
'keyspace_id:"\\001\\002\\003" ]')
key_ranges = [keyrange.KeyRange('01020304-05060708')]
key_ranges_echo = '[start:"\\001\\002\\003\\004" end:"\\005\\006\\007\\010" ]'
tablet_type = 'replica'
tablet_type_echo = 'REPLICA'
bind_variables = {
'int': 123,
'float': 2.1,
'bytes': '\x01\x02\x03',
'bool': True,
}
bind_variables_echo = ('map[bool:type:INT64 value:"1" '
'bytes:type:VARBINARY value:"\\001\\002\\003" '
'float:type:FLOAT64 value:"2.1" '
'int:type:INT64 value:"123" ]')
caller_id = vtgate_client.CallerID(principal='test_principal',
component='test_component',
subcomponent='test_subcomponent')
caller_id_echo = ('principal:"test_principal" component:"test_component"'
' subcomponent:"test_subcomponent" ')
event_token = query_pb2.EventToken(timestamp=123,
shard=shards[0],
position='test_pos')
options_echo = ('include_event_token:true compare_event_token:'
'<timestamp:123 shard:"-80" position:"test_pos" > ')
session_echo = ('autocommit:true target_string:"@replica" '
'options:<include_event_token:'
'true compare_event_token:<timestamp:123 shard:'
'"-80" position:"test_pos" > > ')
def test_echo_execute(self):
"""This test calls the echo method."""
# Execute
cursor = self.conn.cursor(tablet_type=self.tablet_type, keyspace=None)
cursor.set_effective_caller_id(self.caller_id)
cursor.execute(self.echo_prefix + self.query,
self.bind_variables,
include_event_token=True,
compare_event_token=self.event_token)
self._check_echo(
cursor,
{
'callerId': self.caller_id_echo,
# FIXME(alainjobart) change this to query_echo once v3 understand binds
'query': self.echo_prefix + self.query,
'bindVars': self.bind_variables_echo,
'session': self.session_echo,
})
cursor.close()
# ExecuteShards
cursor = self.conn.cursor(tablet_type=self.tablet_type,
keyspace=self.keyspace,
shards=self.shards)
cursor.set_effective_caller_id(self.caller_id)
cursor.execute(self.echo_prefix + self.query,
self.bind_variables,
include_event_token=True,
compare_event_token=self.event_token)
self._check_echo(
cursor, {
'callerId': self.caller_id_echo,
'query': self.echo_prefix + self.query_echo,
'keyspace': self.keyspace,
'shards': self.shards_echo,
'bindVars': self.bind_variables_echo,
'tabletType': self.tablet_type_echo,
'options': self.options_echo,
'fresher': True,
'eventToken': self.event_token,
})
cursor.close()
# ExecuteKeyspaceIds
cursor = self.conn.cursor(tablet_type=self.tablet_type,
keyspace=self.keyspace,
keyspace_ids=self.keyspace_ids)
cursor.set_effective_caller_id(self.caller_id)
cursor.execute(self.echo_prefix + self.query,
self.bind_variables,
include_event_token=True,
compare_event_token=self.event_token)
self._check_echo(
cursor, {
'callerId': self.caller_id_echo,
'query': self.echo_prefix + self.query_echo,
'keyspace': self.keyspace,
'keyspaceIds': self.keyspace_ids_echo,
'bindVars': self.bind_variables_echo,
'tabletType': self.tablet_type_echo,
'options': self.options_echo,
'fresher': True,
'eventToken': self.event_token,
})
cursor.close()
# ExecuteKeyRanges
cursor = self.conn.cursor(tablet_type=self.tablet_type,
keyspace=self.keyspace,
keyranges=self.key_ranges)
cursor.set_effective_caller_id(self.caller_id)
cursor.execute(self.echo_prefix + self.query,
self.bind_variables,
include_event_token=True,
compare_event_token=self.event_token)
self._check_echo(
cursor, {
'callerId': self.caller_id_echo,
'query': self.echo_prefix + self.query_echo,
'keyspace': self.keyspace,
'keyRanges': self.key_ranges_echo,
'bindVars': self.bind_variables_echo,
'tabletType': self.tablet_type_echo,
})
cursor.close()
# ExecuteEntityIds
cursor = self.conn.cursor(tablet_type=self.tablet_type,
keyspace=self.keyspace)
cursor.set_effective_caller_id(self.caller_id)
cursor.execute(self.echo_prefix + self.query,
self.bind_variables,
entity_keyspace_id_map=self.entity_keyspace_ids,
entity_column_name='column1',
include_event_token=True,
compare_event_token=self.event_token)
self._check_echo(
cursor, {
'callerId': self.caller_id_echo,
'query': self.echo_prefix + self.query_echo,
'keyspace': self.keyspace,
'entityColumnName': 'column1',
'entityIds': self.entity_keyspace_ids_echo,
'bindVars': self.bind_variables_echo,
'tabletType': self.tablet_type_echo,
'options': self.options_echo,
'fresher': True,
'eventToken': self.event_token,
})
cursor.close()
# ExecuteBatchShards
cursor = self.conn.cursor(tablet_type=self.tablet_type,
keyspace=None,
as_transaction=True)
cursor.set_effective_caller_id(self.caller_id)
cursor.executemany(sql=None,
params_list=[
dict(sql=self.echo_prefix + self.query,
bind_variables=self.bind_variables,
keyspace=self.keyspace,
shards=self.shards)
])
self._check_echo(
cursor, {
'callerId': self.caller_id_echo,
'query': self.echo_prefix + self.query_echo,
'keyspace': self.keyspace,
'shards': self.shards_echo,
'bindVars': self.bind_variables_echo,
'tabletType': self.tablet_type_echo,
'asTransaction': 'true',
})
cursor.close()
# ExecuteBatchKeyspaceIds
cursor = self.conn.cursor(tablet_type=self.tablet_type,
keyspace=None,
as_transaction=True)
cursor.set_effective_caller_id(self.caller_id)
cursor.executemany(sql=None,
params_list=[
dict(sql=self.echo_prefix + self.query,
bind_variables=self.bind_variables,
keyspace=self.keyspace,
keyspace_ids=self.keyspace_ids)
])
self._check_echo(
cursor, {
'callerId': self.caller_id_echo,
'query': self.echo_prefix + self.query_echo,
'keyspace': self.keyspace,
'keyspaceIds': self.keyspace_ids_echo,
'bindVars': self.bind_variables_echo,
'tabletType': self.tablet_type_echo,
'asTransaction': 'true',
})
cursor.close()
def _get_echo(self, cursor):
result = {}
data = cursor.fetchall()
for i, (n, _) in enumerate(cursor.description):
result[n] = data[0][i]
return result
def _check_echo(self, cursor, values):
"""_check_echo makes sure the echo result is correct."""
got = self._get_echo(cursor)
for k, v in values.iteritems():
if k == 'fresher':
self.assertTrue(self.conn.fresher)
elif k == 'eventToken':
self.assertEqual(
text_format.MessageToString(self.conn.event_token),
text_format.MessageToString(v))
else:
self.assertEqual(
got[k], v, 'item %s is different in result: got %s'
' expected %s' % (k, got[k], v))
# Check NULL and empty string.
self.assertEqual(got['null'], None)
self.assertEqual(got['emptyString'], '')