def test_database_filter(self): start_position = _get_master_current_position() master_tablet.mquery('other_database', _create_vt_insert_test) self._exec_vt_txn(self._populate_vt_insert_test) logging.debug('test_database_filter: starting @ %s', start_position) master_conn = self._get_vtgate_stream_conn() for event, _ in master_conn.update_stream( 'test_keyspace', topodata_pb2.MASTER, event=query_pb2.EventToken(shard='0', position=start_position), shard='0'): for statement in event.statements: self.assertNotEqual(statement.category, 2, # query_pb2.StreamEvent.DDL "query using other_database wasn't filtered out") break master_conn.close()
def test_update_stream_interrupt(self): """Checks that a running query is terminated on going non-serving.""" # Make sure the replica is replica type. utils.run_vtctl( ['ChangeSlaveType', replica_tablet.tablet_alias, 'replica']) logging.debug('sleeping a bit for the replica action to complete') utils.wait_for_tablet_type(replica_tablet.tablet_alias, 'replica', 30) # Save current position, insert some data. start_position = _get_repl_current_position() logging.debug('test_update_stream_interrupt starting @ %s', start_position) self._exec_vt_txn(self._populate_vt_a(1)) self._exec_vt_txn(['delete from vt_a']) # Start an Update Stream from the slave. When we get the data, go to spare. # That should interrupt the streaming RPC. replica_conn = self._get_vtgate_stream_conn() first = True txn_count = 0 try: for event, resume_timestamp in replica_conn.update_stream( 'test_keyspace', topodata_pb2.REPLICA, event=query_pb2.EventToken(shard='0', position=start_position), shard='0'): logging.debug('test_update_stream_interrupt got event(%d): %s', resume_timestamp, event) if first: utils.run_vtctl([ 'ChangeSlaveType', replica_tablet.tablet_alias, 'spare' ]) utils.wait_for_tablet_type(replica_tablet.tablet_alias, 'spare', 30) first = False else: if event.event_token.position: txn_count += 1 self.assertFail('update_stream terminated with no exception') except dbexceptions.DatabaseError as e: self.assertIn('context canceled', str(e)) self.assertFalse(first) logging.debug('Streamed %d transactions before exiting', txn_count) replica_conn.close()
def test_ddl(self): """Asks for all statements since we started, find the DDL.""" start_position = master_start_position logging.debug('test_ddl: starting @ %s', start_position) master_conn = self._get_vtgate_stream_conn() found = False for event, _ in master_conn.update_stream( 'test_keyspace', topodata_pb2.MASTER, event=query_pb2.EventToken(shard='0', position=start_position), shard='0'): for statement in event.statements: if statement.sql == _create_vt_insert_test: found = True break break master_conn.close() self.assertTrue(found, "didn't get right sql")
def test_service_switch(self): """tests the service switch from disable -> enable -> disable.""" # make the replica spare utils.run_vtctl(['ChangeSlaveType', replica_tablet.tablet_alias, 'spare']) utils.wait_for_tablet_type(replica_tablet.tablet_alias, 'spare') # Check UpdateStreamState is disabled. v = utils.get_vars(replica_tablet.port) if v['UpdateStreamState'] != 'Disabled': self.fail("Update stream service should be 'Disabled' but is '%s'" % v['UpdateStreamState']) start_position = _get_repl_current_position() # Make sure we can't start a new request to vttablet directly. _, stderr = utils.run_vtctl(['VtTabletUpdateStream', '-position', start_position, replica_tablet.tablet_alias], expect_fail=True) self.assertIn('operation not allowed in state NOT_SERVING', stderr) # Make sure we can't start a new request through vtgate. replica_conn = self._get_vtgate_stream_conn() try: for event, resume_timestamp in replica_conn.update_stream( 'test_keyspace', topodata_pb2.REPLICA, event=query_pb2.EventToken(shard='0', position=start_position), shard='0'): self.assertFail('got event(%d): %s' % (resume_timestamp, str(event))) self.assertFail('update_stream terminated with no exception') except dbexceptions.DatabaseError as e: self.assertIn(vtgate_gateway_flavor().no_tablet_found_message(), str(e)) # Go back to replica. utils.run_vtctl( ['ChangeSlaveType', replica_tablet.tablet_alias, 'replica']) utils.wait_for_tablet_type(replica_tablet.tablet_alias, 'replica') # Check UpdateStreamState is enabled. v = utils.get_vars(replica_tablet.port) if v['UpdateStreamState'] != 'Enabled': self.fail("Update stream service should be 'Enabled' but is '%s'" % v['UpdateStreamState'])
def test_set_insert_id(self): start_position = _get_master_current_position() self._exec_vt_txn( ['SET INSERT_ID=1000000'] + self._populate_vt_insert_test) logging.debug('test_set_insert_id: starting @ %s', start_position) master_conn = self._get_vtgate_stream_conn() expected_id = 1000000 for event, _ in master_conn.update_stream( 'test_keyspace', topodata_pb2.MASTER, event=query_pb2.EventToken(shard='0', position=start_position), shard='0'): for statement in event.statements: fields, rows = proto3_encoding.convert_stream_event_statement(statement) self.assertEqual(fields[0], 'id') self.assertEqual(rows[0][0], expected_id) expected_id += 1 break if expected_id != 1000004: self.fail('did not get my four values!') master_conn.close()
def test_stream_parity(self): """Tests parity of streams between master and replica for the same writes. Also tests transactions are retrieved properly. """ timeout = 30 while True: master_position = _get_master_current_position() replica_position = _get_repl_current_position() if master_position == replica_position: break timeout = utils.wait_step( '%s == %s' % (master_position, replica_position), timeout) logging.debug('run_test_stream_parity starting @ %s', master_position) self._exec_vt_txn(self._populate_vt_a(15)) self._exec_vt_txn(self._populate_vt_b(14)) self._exec_vt_txn(['delete from vt_a']) self._exec_vt_txn(['delete from vt_b']) # get master events master_conn = self._get_vtgate_stream_conn() master_events = [] for event, resume_timestamp in master_conn.update_stream( 'test_keyspace', topodata_pb2.MASTER, event=query_pb2.EventToken(shard='0', position=master_position), shard='0'): logging.debug('Got master event(%d): %s', resume_timestamp, event) master_events.append(event) if len(master_events) == 4: break master_conn.close() # get replica events replica_conn = self._get_vtgate_stream_conn() replica_events = [] for event, resume_timestamp in replica_conn.update_stream( 'test_keyspace', topodata_pb2.REPLICA, event=query_pb2.EventToken(shard='0', position=replica_position), shard='0'): logging.debug('Got slave event(%d): %s', resume_timestamp, event) replica_events.append(event) if len(replica_events) == 4: break replica_conn.close() # and compare if len(master_events) != len(replica_events): logging.debug( 'Test Failed - # of records mismatch, master %s replica %s', master_events, replica_events) for master_event, replica_event in zip(master_events, replica_events): # The timestamp is from when the event was written to the binlogs. # the master uses the timestamp of when it wrote it originally, # the slave of when it applied the logs. These can differ and make this # test flaky. So we just blank them out, easier. We really want to # compare the replication positions. master_event.event_token.timestamp = 123 replica_event.event_token.timestamp = 123 self.assertEqual( master_event, replica_event, "Test failed, data mismatch - master '%s' and replica '%s'" % (master_event, replica_event)) logging.debug('Test Writes: PASS')
def test_event_token_fresher(self): """event_token.fresher test suite.""" test_cases = [ { 'ev1': None, 'ev2': None, 'expected': -1, }, { 'ev1': query_pb2.EventToken(timestamp=123, ), 'ev2': None, 'expected': -1, }, { 'ev1': None, 'ev2': query_pb2.EventToken(timestamp=123, ), 'expected': -1, }, { 'ev1': query_pb2.EventToken(timestamp=123, ), 'ev2': query_pb2.EventToken(timestamp=123, ), 'expected': -1, }, { 'ev1': query_pb2.EventToken(timestamp=200, ), 'ev2': query_pb2.EventToken(timestamp=100, ), 'expected': 100, }, { 'ev1': query_pb2.EventToken(timestamp=100, ), 'ev2': query_pb2.EventToken(timestamp=200, ), 'expected': -100, }, { # Test cases with not enough information to compare. 'ev1': query_pb2.EventToken(timestamp=100, ), 'ev2': query_pb2.EventToken(timestamp=100, ), 'expected': -1, }, { 'ev1': query_pb2.EventToken( timestamp=100, shard='s1', ), 'ev2': query_pb2.EventToken( timestamp=100, shard='s2', ), 'expected': -1, }, { 'ev1': query_pb2.EventToken( timestamp=100, shard='s1', ), 'ev2': query_pb2.EventToken( timestamp=100, shard='s1', ), 'expected': -1, }, { 'ev1': query_pb2.EventToken( timestamp=100, shard='s1', position='pos1', ), 'ev2': query_pb2.EventToken( timestamp=100, shard='s1', ), 'expected': -1, }, { 'ev1': query_pb2.EventToken( timestamp=100, shard='s1', ), 'ev2': query_pb2.EventToken( timestamp=100, shard='s1', position='pos2', ), 'expected': -1, }, { 'ev1': query_pb2.EventToken( timestamp=100, shard='s1', position='pos1', # invalid on purpose ), 'ev2': query_pb2.EventToken( timestamp=100, shard='s1', position='pos2', # invalid on purpose ), 'expected': -1, }, { 'ev1': query_pb2.EventToken( timestamp=100, shard='s1', position='MariaDB/0-1-123', # valid but different ), 'ev2': query_pb2.EventToken( timestamp=100, shard='s1', position= 'MySQL56/33333333-3333-3333-3333-333333333333:456-789', ), 'expected': -1, }, { # MariaDB test cases. 'ev1': query_pb2.EventToken( timestamp=100, shard='s1', position='MariaDB/0-1-200', ), 'ev2': query_pb2.EventToken( timestamp=100, shard='s1', position='MariaDB/0-1-100', ), 'expected': 100, }, { 'ev1': query_pb2.EventToken( timestamp=100, shard='s1', position='MariaDB/0-1-100', ), 'ev2': query_pb2.EventToken( timestamp=100, shard='s1', position='MariaDB/0-1-200', ), 'expected': -100, }, { 'ev1': query_pb2.EventToken( timestamp=100, shard='s1', position='MariaDB/0-1-100', ), 'ev2': query_pb2.EventToken( timestamp=100, shard='s1', position='MariaDB/0-1-100', ), 'expected': 0, }, { # MySQL56 test cases, not supported yet. 'ev1': query_pb2.EventToken( timestamp=100, shard='s1', position= 'MySQL56/33333333-3333-3333-3333-333333333333:1-200', ), 'ev2': query_pb2.EventToken( timestamp=100, shard='s1', position= 'MySQL56/33333333-3333-3333-3333-333333333333:1-100', ), 'expected': -1, # Should be: 1, }, { 'ev1': query_pb2.EventToken( timestamp=100, shard='s1', position= 'MySQL56/33333333-3333-3333-3333-333333333333:1-100', ), 'ev2': query_pb2.EventToken( timestamp=100, shard='s1', position= 'MySQL56/33333333-3333-3333-3333-333333333333:1-200', ), 'expected': -1, }, { 'ev1': query_pb2.EventToken( timestamp=100, shard='s1', position= 'MySQL56/33333333-3333-3333-3333-333333333333:1-100', ), 'ev2': query_pb2.EventToken( timestamp=100, shard='s1', position= 'MySQL56/33333333-3333-3333-3333-333333333333:1-100', ), 'expected': -1, # Should be: 0, } ] for tcase in test_cases: got = event_token.fresher(tcase['ev1'], tcase['ev2']) self.assertEqual( got, tcase['expected'], 'got %d but expected %d for Fresher(%s, %s)' % (got, tcase['expected'], tcase['ev1'], tcase['ev2']))
class TestEcho(TestPythonClientBase): """Send queries to the server, check the returned result matches.""" echo_prefix = 'echo://' query = ( u'test query with bind variables: :int :float :bytes, unicode: ' u'\u6211\u80fd\u541e\u4e0b\u73bb\u7483\u800c\u4e0d\u50b7\u8eab\u9ad4' ).encode('utf-8') query_echo = ( u'test query with bind variables: :int :float :bytes, unicode: ' u'\u6211\u80fd\u541e\u4e0b\u73bb\u7483\u800c\u4e0d\u50b7\u8eab\u9ad4' ).encode('utf-8') keyspace = 'test_keyspace' shards = ['-80', '80-'] shards_echo = '[-80 80-]' keyspace_ids = ['\x01\x02\x03\x04', '\x05\x06\x07\x08'] keyspace_ids_echo = '[[1 2 3 4] [5 6 7 8]]' # FIXME(alainjobart) using a map for the entities makes it impossible to # guarantee the order of the entities in the query. It is really an API # problem here? For this test, however, I'll just use a single value for now entity_keyspace_ids = { 123: '\x01\x02\x03', # 2.0: '\x04\x05\x06', # '\x01\x02\x03': '\x07\x08\x09', } # entity_keyspace_ids_echo = ('[type:INT64 value:"123" ' # 'keyspace_id:"\\001\\002\\003" ' # 'type:FLOAT64 value:"2" ' # 'keyspace_id:"\\004\\005\\006" ' # 'type:VARBINARY value:"\\001\\002\\003" ' # 'keyspace_id:"\\007\\010\\t" ]') entity_keyspace_ids_echo = ('[type:INT64 value:"123" ' 'keyspace_id:"\\001\\002\\003" ]') key_ranges = [keyrange.KeyRange('01020304-05060708')] key_ranges_echo = '[start:"\\001\\002\\003\\004" end:"\\005\\006\\007\\010" ]' tablet_type = 'replica' tablet_type_echo = 'REPLICA' bind_variables = { 'int': 123, 'float': 2.1, 'bytes': '\x01\x02\x03', 'bool': True, } bind_variables_echo = ('map[bool:type:INT64 value:"1" ' 'bytes:type:VARBINARY value:"\\001\\002\\003" ' 'float:type:FLOAT64 value:"2.1" ' 'int:type:INT64 value:"123" ]') caller_id = vtgate_client.CallerID(principal='test_principal', component='test_component', subcomponent='test_subcomponent') caller_id_echo = ('principal:"test_principal" component:"test_component"' ' subcomponent:"test_subcomponent" ') event_token = query_pb2.EventToken(timestamp=123, shard=shards[0], position='test_pos') options_echo = ('include_event_token:true compare_event_token:' '<timestamp:123 shard:"-80" position:"test_pos" > ') session_echo = ('autocommit:true target_string:"@replica" ' 'options:<include_event_token:' 'true compare_event_token:<timestamp:123 shard:' '"-80" position:"test_pos" > > ') def test_echo_execute(self): """This test calls the echo method.""" # Execute cursor = self.conn.cursor(tablet_type=self.tablet_type, keyspace=None) cursor.set_effective_caller_id(self.caller_id) cursor.execute(self.echo_prefix + self.query, self.bind_variables, include_event_token=True, compare_event_token=self.event_token) self._check_echo( cursor, { 'callerId': self.caller_id_echo, # FIXME(alainjobart) change this to query_echo once v3 understand binds 'query': self.echo_prefix + self.query, 'bindVars': self.bind_variables_echo, 'session': self.session_echo, }) cursor.close() # ExecuteShards cursor = self.conn.cursor(tablet_type=self.tablet_type, keyspace=self.keyspace, shards=self.shards) cursor.set_effective_caller_id(self.caller_id) cursor.execute(self.echo_prefix + self.query, self.bind_variables, include_event_token=True, compare_event_token=self.event_token) self._check_echo( cursor, { 'callerId': self.caller_id_echo, 'query': self.echo_prefix + self.query_echo, 'keyspace': self.keyspace, 'shards': self.shards_echo, 'bindVars': self.bind_variables_echo, 'tabletType': self.tablet_type_echo, 'options': self.options_echo, 'fresher': True, 'eventToken': self.event_token, }) cursor.close() # ExecuteKeyspaceIds cursor = self.conn.cursor(tablet_type=self.tablet_type, keyspace=self.keyspace, keyspace_ids=self.keyspace_ids) cursor.set_effective_caller_id(self.caller_id) cursor.execute(self.echo_prefix + self.query, self.bind_variables, include_event_token=True, compare_event_token=self.event_token) self._check_echo( cursor, { 'callerId': self.caller_id_echo, 'query': self.echo_prefix + self.query_echo, 'keyspace': self.keyspace, 'keyspaceIds': self.keyspace_ids_echo, 'bindVars': self.bind_variables_echo, 'tabletType': self.tablet_type_echo, 'options': self.options_echo, 'fresher': True, 'eventToken': self.event_token, }) cursor.close() # ExecuteKeyRanges cursor = self.conn.cursor(tablet_type=self.tablet_type, keyspace=self.keyspace, keyranges=self.key_ranges) cursor.set_effective_caller_id(self.caller_id) cursor.execute(self.echo_prefix + self.query, self.bind_variables, include_event_token=True, compare_event_token=self.event_token) self._check_echo( cursor, { 'callerId': self.caller_id_echo, 'query': self.echo_prefix + self.query_echo, 'keyspace': self.keyspace, 'keyRanges': self.key_ranges_echo, 'bindVars': self.bind_variables_echo, 'tabletType': self.tablet_type_echo, }) cursor.close() # ExecuteEntityIds cursor = self.conn.cursor(tablet_type=self.tablet_type, keyspace=self.keyspace) cursor.set_effective_caller_id(self.caller_id) cursor.execute(self.echo_prefix + self.query, self.bind_variables, entity_keyspace_id_map=self.entity_keyspace_ids, entity_column_name='column1', include_event_token=True, compare_event_token=self.event_token) self._check_echo( cursor, { 'callerId': self.caller_id_echo, 'query': self.echo_prefix + self.query_echo, 'keyspace': self.keyspace, 'entityColumnName': 'column1', 'entityIds': self.entity_keyspace_ids_echo, 'bindVars': self.bind_variables_echo, 'tabletType': self.tablet_type_echo, 'options': self.options_echo, 'fresher': True, 'eventToken': self.event_token, }) cursor.close() # ExecuteBatchShards cursor = self.conn.cursor(tablet_type=self.tablet_type, keyspace=None, as_transaction=True) cursor.set_effective_caller_id(self.caller_id) cursor.executemany(sql=None, params_list=[ dict(sql=self.echo_prefix + self.query, bind_variables=self.bind_variables, keyspace=self.keyspace, shards=self.shards) ]) self._check_echo( cursor, { 'callerId': self.caller_id_echo, 'query': self.echo_prefix + self.query_echo, 'keyspace': self.keyspace, 'shards': self.shards_echo, 'bindVars': self.bind_variables_echo, 'tabletType': self.tablet_type_echo, 'asTransaction': 'true', }) cursor.close() # ExecuteBatchKeyspaceIds cursor = self.conn.cursor(tablet_type=self.tablet_type, keyspace=None, as_transaction=True) cursor.set_effective_caller_id(self.caller_id) cursor.executemany(sql=None, params_list=[ dict(sql=self.echo_prefix + self.query, bind_variables=self.bind_variables, keyspace=self.keyspace, keyspace_ids=self.keyspace_ids) ]) self._check_echo( cursor, { 'callerId': self.caller_id_echo, 'query': self.echo_prefix + self.query_echo, 'keyspace': self.keyspace, 'keyspaceIds': self.keyspace_ids_echo, 'bindVars': self.bind_variables_echo, 'tabletType': self.tablet_type_echo, 'asTransaction': 'true', }) cursor.close() def _get_echo(self, cursor): result = {} data = cursor.fetchall() for i, (n, _) in enumerate(cursor.description): result[n] = data[0][i] return result def _check_echo(self, cursor, values): """_check_echo makes sure the echo result is correct.""" got = self._get_echo(cursor) for k, v in values.iteritems(): if k == 'fresher': self.assertTrue(self.conn.fresher) elif k == 'eventToken': self.assertEqual( text_format.MessageToString(self.conn.event_token), text_format.MessageToString(v)) else: self.assertEqual( got[k], v, 'item %s is different in result: got %s' ' expected %s' % (k, got[k], v)) # Check NULL and empty string. self.assertEqual(got['null'], None) self.assertEqual(got['emptyString'], '')