#!/usr/bin/python # # Copyright 2013, Google Inc. All rights reserved. # Use of this source code is governed by a BSD-style license that can # be found in the LICENSE file. # FIXME(alainjobart) This test does not pass. It is a work in progress # for resharding. import utils import tablet # initial shards # range "" - 80 shard_0_master = tablet.Tablet() shard_0_replica = tablet.Tablet() # range 80 - "" shard_1_master = tablet.Tablet() shard_1_replica = tablet.Tablet() # split shards # range 80 - C0 shard_2_master = tablet.Tablet() shard_2_replica = tablet.Tablet() # range C0 - "" shard_3_master = tablet.Tablet() shard_3_replica = tablet.Tablet() def setup(): utils.zk_setup()
#!/usr/bin/env python
import logging
import time
import unittest
from vtproto import topodata_pb2
import environment
import utils
import tablet
from mysql_flavor import mysql_flavor
from protocols_flavor import protocols_flavor
tablet_62344 = tablet.Tablet(62344)
tablet_62044 = tablet.Tablet(62044)
tablet_41983 = tablet.Tablet(41983)
tablet_31981 = tablet.Tablet(31981)
def setUpModule():
try:
environment.topo_server().setup()
# start mysql instance external to the test
setup_procs = [
tablet_62344.init_mysql(),
tablet_62044.init_mysql(),
tablet_41983.init_mysql(),
tablet_31981.init_mysql(),
]
def setUpModule():
global new_init_db, db_credentials_file
global tablet_master, tablet_replica1, tablet_replica2
tablet_master = tablet.Tablet(use_mysqlctld=use_mysqlctld,
vt_dba_passwd='VtDbaPass')
tablet_replica1 = tablet.Tablet(use_mysqlctld=use_mysqlctld,
vt_dba_passwd='VtDbaPass')
tablet_replica2 = tablet.Tablet(use_mysqlctld=use_mysqlctld,
vt_dba_passwd='VtDbaPass')
try:
environment.topo_server().setup()
credentials = {
'vt_dba': ['VtDbaPass'],
'vt_app': ['VtAppPass'],
'vt_allprivs': ['VtAllprivsPass'],
'vt_repl': ['VtReplPass'],
'vt_filtered': ['VtFilteredPass'],
}
db_credentials_file = environment.tmproot + '/db_credentials.json'
with open(db_credentials_file, 'w') as fd:
fd.write(json.dumps(credentials))
# Determine which column is used for user passwords in this MySQL version.
proc = tablet_master.init_mysql()
if use_mysqlctld:
tablet_master.wait_for_mysqlctl_socket()
else:
utils.wait_procs([proc])
try:
tablet_master.mquery('mysql',
'select password from mysql.user limit 0',
user='******')
password_col = 'password'
except MySQLdb.DatabaseError:
password_col = 'authentication_string'
utils.wait_procs([tablet_master.teardown_mysql()])
tablet_master.remove_tree(ignore_options=True)
# Create a new init_db.sql file that sets up passwords for all users.
# Then we use a db-credentials-file with the passwords.
new_init_db = environment.tmproot + '/init_db_with_passwords.sql'
with open(environment.vtroot + '/config/init_db.sql') as fd:
init_db = fd.read()
with open(new_init_db, 'w') as fd:
fd.write(init_db)
fd.write(mysql_flavor().change_passwords(password_col))
# start mysql instance external to the test
setup_procs = [
tablet_master.init_mysql(
init_db=new_init_db,
extra_args=['-db-credentials-file', db_credentials_file]),
tablet_replica1.init_mysql(
init_db=new_init_db,
extra_args=['-db-credentials-file', db_credentials_file]),
tablet_replica2.init_mysql(
init_db=new_init_db,
extra_args=['-db-credentials-file', db_credentials_file]),
]
if use_mysqlctld:
tablet_master.wait_for_mysqlctl_socket()
tablet_replica1.wait_for_mysqlctl_socket()
tablet_replica2.wait_for_mysqlctl_socket()
else:
utils.wait_procs(setup_procs)
except:
tearDownModule()
raise
import json
import logging
import os
import socket
import unittest
import urllib2
import re
import environment
import tablet
import utils
from zk import zkocc
from vtctl import vtctl_client
# range '' - 80
shard_0_master = tablet.Tablet()
shard_0_replica = tablet.Tablet()
shard_0_spare = tablet.Tablet()
# range 80 - ''
shard_1_master = tablet.Tablet()
shard_1_replica = tablet.Tablet()
# not assigned
idle = tablet.Tablet()
scrap = tablet.Tablet()
# all tablets
tablets = [
shard_0_master, shard_0_replica, shard_1_master, shard_1_replica, idle,
scrap, shard_0_spare
]
import warnings
# Dropping a table inexplicably produces a warning despite
# the "IF EXISTS" clause. Squelch these warnings.
warnings.simplefilter('ignore')
import logging
import unittest
import environment
import tablet
import utils
use_mysqlctld = True
tablet_master = tablet.Tablet(use_mysqlctld=use_mysqlctld)
tablet_replica1 = tablet.Tablet(use_mysqlctld=use_mysqlctld)
tablet_replica2 = tablet.Tablet(use_mysqlctld=use_mysqlctld)
setup_procs = []
def setUpModule():
try:
environment.topo_server().setup()
# start mysql instance external to the test
global setup_procs
setup_procs = [
tablet_master.init_mysql(),
tablet_replica1.init_mysql(),
import environment import tablet import utils from vtdb import dbexceptions from vtdb import proto3_encoding from vtdb import vtgate_client from vtproto import query_pb2 from vtproto import topodata_pb2 from mysql_flavor import mysql_flavor from protocols_flavor import protocols_flavor from vtgate_gateway_flavor.gateway import vtgate_gateway_flavor # global flag to control which type of replication we use. use_rbr = False master_tablet = tablet.Tablet() replica_tablet = tablet.Tablet() # master_start_position has the replication position before we start # doing anything to the master database. It is used by test_ddl to # make sure we see DDLs. master_start_position = None _create_vt_insert_test = '''create table if not exists vt_insert_test ( id bigint auto_increment, msg varchar(64), primary key (id) ) Engine=InnoDB''' _create_vt_a = '''create table if not exists vt_a ( eid bigint,
import time import logging import unittest import base_sharding import environment import tablet import utils from vtproto import topodata_pb2 from vtdb import keyrange_constants # initial shards # range '' - 80 shard_0_master = tablet.Tablet() shard_0_replica = tablet.Tablet() shard_0_ny_rdonly = tablet.Tablet(cell='ny') # range 80 - '' shard_1_master = tablet.Tablet() shard_1_slave1 = tablet.Tablet() shard_1_slave2 = tablet.Tablet() shard_1_ny_rdonly = tablet.Tablet(cell='ny') shard_1_rdonly1 = tablet.Tablet() # split shards # range 80 - c0 shard_2_master = tablet.Tablet() shard_2_replica1 = tablet.Tablet() shard_2_replica2 = tablet.Tablet() shard_2_rdonly1 = tablet.Tablet()
# Dropping a table inexplicably produces a warning despite
# the "IF EXISTS" clause. Squelch these warnings.
warnings.simplefilter("ignore")
import gzip
import logging
import os
import shutil
from subprocess import call
import unittest
import environment
import utils
import tablet
tablet_62344 = tablet.Tablet(62344)
tablet_31981 = tablet.Tablet(31981)
def setUpModule():
try:
environment.topo_server_setup()
# start mysql instance external to the test
setup_procs = [
tablet_62344.init_mysql(),
tablet_31981.init_mysql(),
]
utils.wait_procs(setup_procs)
except:
tearDownModule()
import unittest
import base_sharding
import environment
import tablet
import utils
from vtproto import topodata_pb2
from vtdb import keyrange_constants
from vtdb import vtgate_client
use_alias = True
# initial shards
# range '' - 80
shard_0_master = tablet.Tablet()
shard_0_replica = tablet.Tablet(cell='ny')
shard_0_rdonly = tablet.Tablet(cell='ny')
#shard_0_replica = tablet.Tablet()
#shard_0_rdonly = tablet.Tablet()
# range 80 - ''
shard_1_master = tablet.Tablet()
#shard_1_replica = tablet.Tablet()
#shard_1_rdonly = tablet.Tablet()
shard_1_replica = tablet.Tablet(cell='ny')
shard_1_rdonly = tablet.Tablet(cell='ny')
all_tablets = ([shard_0_master, shard_0_replica, shard_0_rdonly,
shard_1_master, shard_1_replica,shard_1_rdonly])
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests that VTGate buffers master traffic when expected."""
import logging
import struct
import unittest
from vtdb import keyrange
from vtdb import vtgate_client
import environment
import tablet
import utils
shard_0_master = tablet.Tablet()
shard_0_replica1 = tablet.Tablet()
KEYSPACE_NAME = 'test_keyspace'
SHARD_NAMES = ['0']
SHARD_KID_MAP = {
'0': [
527875958493693904, 626750931627689502, 345387386794260318,
332484755310826578, 1842642426274125671, 1326307661227634652,
1761124146422844620, 1661669973250483744, 3361397649937244239,
2444880764308344533, 9767889778372766922, 9742070682920810358,
10296850775085416642, 9537430901666854108, 10440455099304929791,
11454183276974683945, 11185910247776122031, 10460396697869122981,
13379616110062597001, 12826553979133932576
],
}
import json
import logging
import re
import time
import unittest
import urllib2
from vtproto import topodata_pb2
import environment
import tablet
import utils
# range '' - 80
shard_0_master = tablet.Tablet(use_mysqlctld=True)
shard_0_replica = tablet.Tablet(use_mysqlctld=True)
shard_0_rdonly = tablet.Tablet(use_mysqlctld=True)
all_shard_0_tablets = [shard_0_master, shard_0_replica, shard_0_rdonly]
# range 80 - ''
shard_1_master = tablet.Tablet(use_mysqlctld=True)
shard_1_replica = tablet.Tablet(use_mysqlctld=True)
shard_1_rdonly = tablet.Tablet(use_mysqlctld=True)
all_shard_1_tablets = [shard_1_master, shard_1_replica, shard_1_rdonly]
# all tablets
all_tablets = all_shard_0_tablets + all_shard_1_tablets
def setUpModule():
try:
environment.topo_server().setup()
class TestServiceTestOfTabletManager(unittest.TestCase):
"""This tests the locking functionality by running rpc calls against the tabletmanager,
in contrast to testing the tabletmanager through the vtcl"""
replica = tablet.Tablet(62344)
master = tablet.Tablet(62044)
def setUp(self):
try:
os.makedirs(environment.tmproot)
except OSError:
# directory already exists
pass
try:
topo_flavor = environment.topo_server().flavor()
environment.topo_server().setup()
# start mysql instance external to the test
setup_procs = [
self.replica.init_mysql(),
self.master.init_mysql(),
]
utils.Vtctld().start()
logging.debug(utils.vtctld_connection)
utils.wait_procs(setup_procs)
for t in self.master, self.replica:
t.create_db('vt_test_keyspace')
self.master.init_tablet('replica',
'test_keyspace',
'0',
start=True)
self.replica.init_tablet('replica',
'test_keyspace',
'0',
start=True)
utils.run_vtctl([
'InitShardMaster', '-force', 'test_keyspace/0',
self.master.tablet_alias
])
self.master.mquery('vt_test_keyspace', _create_vt_insert_test)
for t in [self.master, self.replica]:
t.set_semi_sync_enabled(master=False, slave=False)
except Exception as e:
logging.exception(e)
self.tearDown()
def tearDown(self):
try:
for t in self.master, self.replica:
t.kill_vttablet()
tablet.Tablet.check_vttablet_count()
environment.topo_server().wipe()
for t in [self.master, self.replica]:
t.reset_replication()
t.set_semi_sync_enabled(master=False, slave=False)
t.clean_dbs()
finally:
utils.required_teardown()
if utils.options.skip_teardown:
return
teardown_procs = [
self.master.teardown_mysql(),
self.replica.teardown_mysql(),
]
utils.wait_procs(teardown_procs, raise_on_error=False)
environment.topo_server().teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
self.replica.remove_tree()
self.master.remove_tree()
def _write_data_to_master(self):
"""Write a single row to the master"""
self.master.mquery('vt_test_keyspace',
"insert into vt_insert_test (msg) values ('test')",
write=True)
def _check_data_on_replica(self, count, msg):
"""Check that the specified tablet has the expected number of rows."""
timeout = 3
while True:
try:
result = self.replica.mquery(
'vt_test_keyspace', 'select count(*) from vt_insert_test')
if result[0][0] == count:
break
except MySQLdb.DatabaseError:
# ignore exceptions, we'll just timeout (the tablet creation
# can take some time to replicate, and we get a 'table vt_insert_test
# does not exist exception in some rare cases)
logging.exception('exception waiting for data to replicate')
timeout = utils.wait_step(msg, timeout)
def test_lock_and_unlock(self):
"""Test the lock ability by locking a replica and asserting it does not see changes"""
# first make sure that our writes to the master make it to the replica
self._write_data_to_master()
self._check_data_on_replica(1, "replica getting the data")
# now lock the replica
tablet_manager = self.replica.tablet_manager()
tablet_manager.LockTables(LockTablesRequest())
# make sure that writing to the master does not show up on the replica while locked
self._write_data_to_master()
with self.assertRaises(TestError):
self._check_data_on_replica(
2, "the replica should not see these updates")
# finally, make sure that unlocking the replica leads to the previous write showing up
tablet_manager.UnlockTables(UnlockTablesRequest())
self._check_data_on_replica(
2, "after unlocking the replica, we should see these updates")
def test_unlock_when_we_dont_have_a_lock(self):
"""Unlocking when we do not have a valid lock should lead to an exception being raised"""
# unlock the replica
tablet_manager = self.replica.tablet_manager()
with self.assertRaises(Exception):
tablet_manager.UnlockTables(UnlockTablesRequest())
def test_start_slave_until_after(self):
"""Test by writing three rows, noting the gtid after each, and then replaying them one by one"""
self.replica.start_vttablet()
self.master.start_vttablet()
# first we stop replication to the replica, so we can move forward step by step.
replica_tablet_manager = self.replica.tablet_manager()
replica_tablet_manager.StopSlave(StopSlaveRequest())
master_tablet_manager = self.master.tablet_manager()
self._write_data_to_master()
pos1 = master_tablet_manager.MasterPosition(MasterPositionRequest())
self._write_data_to_master()
pos2 = master_tablet_manager.MasterPosition(MasterPositionRequest())
self._write_data_to_master()
pos3 = master_tablet_manager.MasterPosition(MasterPositionRequest())
# Now, we'll resume stepwise position by position and make sure that we see the expected data
self._check_data_on_replica(0, "no data has yet reached the replica")
# timeout is given in nanoseconds. we want to wait no more than 10 seconds
timeout = int(10 * 1e9)
replica_tablet_manager.StartSlaveUntilAfter(
StartSlaveUntilAfterRequest(position=pos1.position,
wait_timeout=timeout))
self._check_data_on_replica(1, "first row is now visible")
replica_tablet_manager.StartSlaveUntilAfter(
StartSlaveUntilAfterRequest(position=pos2.position,
wait_timeout=timeout))
self._check_data_on_replica(2, "second row is now visible")
replica_tablet_manager.StartSlaveUntilAfter(
StartSlaveUntilAfterRequest(position=pos3.position,
wait_timeout=timeout))
self._check_data_on_replica(3, "third row is now visible")
def test_lock_and_timeout(self):
"""Test that the lock times out and updates can be seen even though nothing is unlocked"""
# first make sure that our writes to the master make it to the replica
self._write_data_to_master()
self._check_data_on_replica(1, "replica getting the data")
# now lock the replica
tablet_manager = self.replica.tablet_manager()
tablet_manager.LockTables(LockTablesRequest())
# make sure that writing to the master does not show up on the replica while locked
self._write_data_to_master()
with self.assertRaises(TestError):
self._check_data_on_replica(
2, "the replica should not see these updates")
# the tests sets the lock timeout to 5 seconds, so sleeping 10 should be safe
time.sleep(10)
self._check_data_on_replica(
2, "the replica should now see these updates")
# finally, trying to unlock should clearly tell us we did not have the lock
with self.assertRaises(Exception):
tablet_manager.UnlockTables(UnlockTablesRequest())
#!/usr/bin/env python
import json
import logging
import os
import unittest
import MySQLdb
import environment
import tablet
import utils
use_mysqlctld = True
tablet_master = tablet.Tablet(use_mysqlctld=use_mysqlctld,
vt_dba_passwd='VtDbaPass')
tablet_replica1 = tablet.Tablet(use_mysqlctld=use_mysqlctld,
vt_dba_passwd='VtDbaPass')
tablet_replica2 = tablet.Tablet(use_mysqlctld=use_mysqlctld,
vt_dba_passwd='VtDbaPass')
new_init_db = ''
db_credentials_file = ''
def setUpModule():
global new_init_db, db_credentials_file
try:
environment.topo_server().setup()
class VttabletTestEnv(TestEnv):
tablet = tablet.Tablet(62344, 9461, 9460)
vttop = os.getenv("VTTOP")
vtroot = os.getenv("VTROOT")
def setUp(self):
utils.zk_setup()
environment.setup()
if self.vttop is None:
raise EnvironmentError("VTTOP not defined")
if self.vtroot is None:
raise EnvironmentError("VTROOT not defined")
framework.execute('go install', verbose=utils.options.verbose, cwd=self.vttop+'/go/cmd/mysqlctl')
utils.wait_procs([self.tablet.init_mysql()])
self.tablet.mquery("", ["create database vt_test_keyspace", "set global read_only = off"])
self.mysql_conn, mcu = self.tablet.connect('vt_test_keyspace')
self.clean_sqls = []
self.init_sqls = []
clean_mode = False
with open(os.path.join(self.vttop, "test", "test_data", "test_schema.sql")) as f:
for line in f:
line = line.rstrip()
if line == "# clean":
clean_mode = True
if line=='' or line.startswith("#"):
continue
if clean_mode:
self.clean_sqls.append(line)
else:
self.init_sqls.append(line)
try:
for line in self.init_sqls:
mcu.execute(line, {})
finally:
mcu.close()
utils.run_vtctl('CreateKeyspace -force /zk/global/vt/keyspaces/test_keyspace')
self.tablet.init_tablet('master', 'test_keyspace', '0')
customrules = '/tmp/customrules.json'
self.create_customrules(customrules)
schema_override = '/tmp/schema_override.json'
self.create_schema_override(schema_override)
if utils.options.memcache:
self.tablet.start_vttablet(memcache=True, customrules=customrules, schema_override=schema_override)
else:
self.tablet.start_vttablet(customrules=customrules, schema_override=schema_override)
# FIXME(szopa): This is necessary here only because of a bug that
# makes the qs reload its config only after an action.
utils.run_vtctl('Ping ' + self.tablet.zk_tablet_path)
for i in range(30):
try:
self.conn = self.connect()
self.txlogger = subprocess.Popen(['curl', '-s', '-N', 'http://localhost:9461/debug/txlog'], stdout=open('/tmp/vtocc_txlog.log', 'w'))
self.txlog = framework.Tailer(open('/tmp/vtocc_txlog.log'), flush=self.tablet.flush)
self.log = framework.Tailer(open(os.path.join(environment.vtlogroot, 'vttablet.INFO')), flush=self.tablet.flush)
querylog_file = '/tmp/vtocc_streamlog_%s.log' % self.tablet.port
utils.run_bg(['curl', '-s', '-N', 'http://localhost:9461/debug/querylog?full=true'], stdout=open(querylog_file, 'w'))
time.sleep(1)
self.querylog = framework.Tailer(open(querylog_file), sleep=0.1)
return
except dbexceptions.OperationalError:
if i == 29:
raise
time.sleep(1)
def tearDown(self):
self.tablet.kill_vttablet()
try:
utils.wait_procs([self.tablet.teardown_mysql()])
except:
# FIXME: remove
pass
if getattr(self, "txlogger", None):
self.txlogger.terminate()
utils.zk_teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
# self.tablet.remove_tree()
def mysql_connect(self, dbname=''):
return self.tablet.connect()
# This file contains integration tests for the go/vt/binlog package.
# It sets up filtered replication between two shards and checks how data flows
# through binlog streamer.
import logging
import unittest
import environment
import tablet
import utils
from mysql_flavor import mysql_flavor
from vtdb import keyrange_constants
from vtdb import update_stream_service
src_master = tablet.Tablet()
src_replica = tablet.Tablet()
dst_master = tablet.Tablet()
dst_replica = tablet.Tablet()
def setUpModule():
try:
environment.topo_server_setup()
setup_procs = [
src_master.init_mysql(),
src_replica.init_mysql(),
dst_master.init_mysql(),
dst_replica.init_mysql(),
]
warnings.simplefilter("ignore")
import gzip
import logging
import os
import shutil
from subprocess import call
import unittest
import environment
import utils
import tablet
use_mysqlctld = True
tablet_62344 = tablet.Tablet(62344, use_mysqlctld=use_mysqlctld)
tablet_31981 = tablet.Tablet(31981, use_mysqlctld=use_mysqlctld)
def setUpModule():
try:
environment.topo_server().setup()
# start mysql instance external to the test
global setup_procs
setup_procs = [
tablet_62344.init_mysql(),
tablet_31981.init_mysql(),
]
if use_mysqlctld:
tablet_62344.wait_for_mysqlctl_socket()
def setUpModule():
xtrabackup_args = [
'-backup_engine_implementation', 'xtrabackup',
'-xtrabackup_stream_mode', stream_mode, '-xtrabackup_user=vt_dba',
'-xtrabackup_backup_flags', '--password=VtDbaPass'
]
global new_init_db, db_credentials_file
global tablet_master, tablet_replica1, tablet_replica2
tablet_master = tablet.Tablet(use_mysqlctld=use_mysqlctld,
vt_dba_passwd='VtDbaPass')
tablet_replica1 = tablet.Tablet(use_mysqlctld=use_mysqlctld,
vt_dba_passwd='VtDbaPass')
tablet_replica2 = tablet.Tablet(use_mysqlctld=use_mysqlctld,
vt_dba_passwd='VtDbaPass')
try:
environment.topo_server().setup()
# Create a new init_db.sql file that sets up passwords for all users.
# Then we use a db-credentials-file with the passwords.
new_init_db = environment.tmproot + '/init_db_with_passwords.sql'
with open(environment.vttop + '/config/init_db.sql') as fd:
init_db = fd.read()
with open(new_init_db, 'w') as fd:
fd.write(init_db)
fd.write('''
# Set real passwords for all users.
ALTER USER 'root'@'localhost' IDENTIFIED BY 'RootPass';
ALTER USER 'vt_dba'@'localhost' IDENTIFIED BY 'VtDbaPass';
ALTER USER 'vt_app'@'localhost' IDENTIFIED BY 'VtAppPass';
ALTER USER 'vt_allprivs'@'localhost' IDENTIFIED BY 'VtAllPrivsPass';
ALTER USER 'vt_repl'@'%' IDENTIFIED BY 'VtReplPass';
ALTER USER 'vt_filtered'@'localhost' IDENTIFIED BY 'VtFilteredPass';
FLUSH PRIVILEGES;
''')
credentials = {
'vt_dba': ['VtDbaPass'],
'vt_app': ['VtAppPass'],
'vt_allprivs': ['VtAllprivsPass'],
'vt_repl': ['VtReplPass'],
'vt_filtered': ['VtFilteredPass'],
}
db_credentials_file = environment.tmproot + '/db_credentials.json'
with open(db_credentials_file, 'w') as fd:
fd.write(json.dumps(credentials))
# start mysql instance external to the test
setup_procs = [
tablet_master.init_mysql(
init_db=new_init_db,
extra_args=['-db-credentials-file', db_credentials_file]),
tablet_replica1.init_mysql(
init_db=new_init_db,
extra_args=['-db-credentials-file', db_credentials_file]),
tablet_replica2.init_mysql(
init_db=new_init_db,
extra_args=['-db-credentials-file', db_credentials_file]),
]
if use_mysqlctld:
tablet_master.wait_for_mysqlctl_socket()
tablet_replica1.wait_for_mysqlctl_socket()
tablet_replica2.wait_for_mysqlctl_socket()
else:
utils.wait_procs(setup_procs)
except:
tearDownModule()
raise
# # Copyright 2013, Google Inc. All rights reserved. # Use of this source code is governed by a BSD-style license that can # be found in the LICENSE file. import logging import threading import time import unittest import utils import tablet # initial shards # range "" - 80 shard_0_master = tablet.Tablet() shard_0_replica = tablet.Tablet() # range 80 - "" shard_1_master = tablet.Tablet() shard_1_slave1 = tablet.Tablet() shard_1_slave2 = tablet.Tablet() # split shards # range 80 - C0 shard_2_master = tablet.Tablet() shard_2_replica1 = tablet.Tablet() shard_2_replica2 = tablet.Tablet() # range C0 - "" shard_3_master = tablet.Tablet() shard_3_replica = tablet.Tablet()
import threading import time import traceback import unittest import urllib import environment import keyspace_util import tablet import utils from vtdb import cursorv3 from vtdb import dbexceptions from vtdb import vtgatev3 shard_0_master = tablet.Tablet() shard_1_master = tablet.Tablet() lookup_master = tablet.Tablet() keyspace_env = None create_vt_user = '''create table vt_user ( id bigint, name varchar(64), primary key (id) ) Engine=InnoDB''' create_vt_user2 = '''create table vt_user2 ( id bigint, name varchar(64), primary key (id)
# This file contains integration tests for the go/vt/binlog package.
# It sets up filtered replication between two shards and checks how data flows
# through binlog streamer.
import logging
import unittest
import environment
import tablet
import utils
from mysql_flavor import mysql_flavor
from vtdb import keyrange_constants
from vtdb import update_stream_service
src_master = tablet.Tablet()
src_replica = tablet.Tablet()
src_rdonly1 = tablet.Tablet()
src_rdonly2 = tablet.Tablet()
dst_master = tablet.Tablet()
dst_replica = tablet.Tablet()
def setUpModule():
try:
environment.topo_server().setup()
setup_procs = [
src_master.init_mysql(),
src_replica.init_mysql(),
src_rdonly1.init_mysql(),
# be found in the LICENSE file.
import unittest
from vtdb import vtgate_client
import environment
import tablet
import utils
SHARDED_KEYSPACE = 'TEST_KEYSPACE_SHARDED'
UNSHARDED_KEYSPACE = 'TEST_KEYSPACE_UNSHARDED'
# shards for SHARDED_KEYSPACE
# range '' - 80
shard_0_master = tablet.Tablet()
shard_0_replica = tablet.Tablet()
# range 80 - ''
shard_1_master = tablet.Tablet()
shard_1_replica = tablet.Tablet()
# shard for UNSHARDED_KEYSPACE
unsharded_master = tablet.Tablet()
unsharded_replica = tablet.Tablet()
shard_names = ['-80', '80-']
shard_kid_map = {
'-80': [
527875958493693904, 626750931627689502, 345387386794260318,
332484755310826578, 1842642426274125671, 1326307661227634652,
1761124146422844620, 1661669973250483744, 3361397649937244239,
import signal
from subprocess import PIPE
import threading
import time
import unittest
import urllib
import urllib2
import environment
import utils
import tablet
from mysql_flavor import mysql_flavor
from protocols_flavor import protocols_flavor
from vtdb import dbexceptions
tablet_62344 = tablet.Tablet(62344)
tablet_62044 = tablet.Tablet(62044)
def setUpModule():
try:
if environment.topo_server().flavor() == 'zookeeper':
# this is a one-off test to make sure our zookeeper implementation
# behaves with a server that is not DNS-resolveable
environment.topo_server().setup(add_bad_host=True)
else:
environment.topo_server().setup()
# start mysql instance external to the test
setup_procs = [
tablet_62344.init_mysql(),
tablet_62044.init_mysql(),
from zk import zkocc
from vtdb import topology
from vtdb import vtclient
import environment
import utils
import tablet
TABLET = "tablet"
VTGATE = "vtgate"
VTGATE_PROTOCOL_TABLET = 'v0'
client_type = TABLET
use_clone_worker = False
# source keyspace, with 4 tables
source_master = tablet.Tablet()
source_replica = tablet.Tablet()
source_rdonly = tablet.Tablet()
# destination keyspace, with just two tables
destination_master = tablet.Tablet()
destination_replica = tablet.Tablet()
destination_rdonly = tablet.Tablet()
def setUpModule():
try:
environment.topo_server().setup()
setup_procs = [
source_master.init_mysql(),
""" import logging import unittest from vtdb import keyrange_constants import base_sharding import environment import tablet import utils # initial shards # shard -40 shard_0_master = tablet.Tablet() shard_0_replica = tablet.Tablet() shard_0_rdonly = tablet.Tablet() # shard 40-80 shard_1_master = tablet.Tablet() shard_1_replica = tablet.Tablet() shard_1_rdonly = tablet.Tablet() # shard 80- shard_2_master = tablet.Tablet() shard_2_replica = tablet.Tablet() shard_2_rdonly = tablet.Tablet() # merged shard -80 shard_dest_master = tablet.Tablet() shard_dest_replica = tablet.Tablet() shard_dest_rdonly = tablet.Tablet()
"""Ensures the vtgate MySQL server protocol plugin works as expected.
We use table ACLs to verify the user name authenticated by the connector is
set properly.
"""
import unittest
import MySQLdb
import environment
import utils
import tablet
# single shard / 2 tablets
shard_0_master = tablet.Tablet()
shard_0_slave = tablet.Tablet()
table_acl_config = environment.tmproot + '/table_acl_config.json'
mysql_auth_server_static = (environment.tmproot +
'/mysql_auth_server_static.json')
def setUpModule():
try:
environment.topo_server().setup()
# setup all processes
setup_procs = [
shard_0_master.init_mysql(),
shard_0_slave.init_mysql(),
'UPDATE %d failed during COMMIT. But we cannot buffer'
' this error and and ignore it. err: %s', attempt, str(e))
else:
self._commit_errors += 1
if self._commit_errors > 1:
raise
logging.debug(
'UPDATE %d failed during COMMIT. This is okay once'
' because we do not support buffering it. err: %s',
attempt, str(e))
def commit_errors(self):
return self._commit_errors
master = tablet.Tablet()
replica = tablet.Tablet()
all_tablets = [master, replica]
def setUpModule():
try:
environment.topo_server().setup()
setup_procs = [t.init_mysql() for t in all_tablets]
utils.Vtctld().start()
utils.wait_procs(setup_procs)
utils.run_vtctl(['CreateKeyspace', KEYSPACE])
# Start tablets.
"""Returns the first replica Tablet instance for the shard, or None."""
if self.replicas:
return self.replicas[0]
else:
return None
@property
def rdonly(self):
"""Returns the first replica Tablet instance for the shard, or None."""
if self.rdonlys:
return self.rdonlys[0]
else:
return None
# initial shard, covers everything
shard_master = tablet.Tablet()
shard_replica = tablet.Tablet()
shard_rdonly1 = tablet.Tablet()
# split shards
# range '' - 80
shard_0_master = tablet.Tablet()
shard_0_replica = tablet.Tablet()
shard_0_rdonly1 = tablet.Tablet()
# range 80 - ''
shard_1_master = tablet.Tablet()
shard_1_replica = tablet.Tablet()
shard_1_rdonly1 = tablet.Tablet()
shard_tablets = ShardTablets(shard_master, [shard_replica], [shard_rdonly1])
shard_0_tablets = ShardTablets(
# Use of this source code is governed by a BSD-style license that can
# be found in the LICENSE file.
import base64
import unittest
from vtproto import topodata_pb2
from vtdb import vtgate_client
import environment
import tablet
import utils
# shards
shard_0_master = tablet.Tablet()
shard_0_rdonly = tablet.Tablet()
shard_1_master = tablet.Tablet()
shard_1_rdonly = tablet.Tablet()
def setUpModule():
try:
environment.topo_server().setup()
setup_procs = [
shard_0_master.init_mysql(),
shard_0_rdonly.init_mysql(),
shard_1_master.init_mysql(),
shard_1_rdonly.init_mysql(),
#!/usr/bin/env python
import os
import logging
import unittest
import environment
import tablet
import utils
shard_0_master = tablet.Tablet()
shard_0_replica1 = tablet.Tablet()
shard_0_replica2 = tablet.Tablet()
shard_0_rdonly = tablet.Tablet()
shard_0_backup = tablet.Tablet()
shard_1_master = tablet.Tablet()
shard_1_replica1 = tablet.Tablet()
shard_2_master = tablet.Tablet()
shard_2_replica1 = tablet.Tablet()
# shard_2 tablets shouldn't exist yet when _apply_initial_schema() is called.
initial_tablets = [
shard_0_master, shard_0_replica1, shard_0_replica2, shard_0_rdonly,
shard_0_backup, shard_1_master, shard_1_replica1,
]
shard_2_tablets = [shard_2_master, shard_2_replica1]
all_tablets = initial_tablets + shard_2_tablets
test_keyspace = 'test_keyspace'
db_name = 'vt_' + test_keyspace
class VttabletTestEnv(TestEnv):
tablet = tablet.Tablet(62344)
vttop = environment.vttop
vtroot = environment.vtroot
@property
def port(self):
return self.tablet.port
def setUp(self):
environment.topo_server_setup()
utils.wait_procs([self.tablet.init_mysql()])
self.tablet.mquery(
"",
["create database vt_test_keyspace", "set global read_only = off"])
self.mysql_conn, mcu = self.tablet.connect('vt_test_keyspace')
self.clean_sqls = []
self.init_sqls = []
clean_mode = False
with open(
os.path.join(self.vttop, "test", "test_data",
"test_schema.sql")) as f:
for line in f:
line = line.rstrip()
if line == "# clean":
clean_mode = True
if line == '' or line.startswith("#"):
continue
if clean_mode:
self.clean_sqls.append(line)
else:
self.init_sqls.append(line)
try:
for line in self.init_sqls:
mcu.execute(line, {})
finally:
mcu.close()
utils.run_vtctl('CreateKeyspace -force test_keyspace')
self.tablet.init_tablet('master', 'test_keyspace', '0')
customrules = os.path.join(environment.tmproot, 'customrules.json')
self.create_customrules(customrules)
schema_override = os.path.join(environment.tmproot,
'schema_override.json')
self.create_schema_override(schema_override)
table_acl_config = os.path.join(environment.vttop, 'test', 'test_data',
'table_acl_config.json')
self.tablet.start_vttablet(
memcache=self.memcache,
customrules=customrules,
schema_override=schema_override,
table_acl_config=table_acl_config,
auth=True,
)
# FIXME(szopa): This is necessary here only because of a bug that
# makes the qs reload its config only after an action.
utils.run_vtctl('Ping ' + self.tablet.tablet_alias)
for i in range(30):
try:
self.conn = self.connect()
self.txlogger = utils.curl(self.url('/debug/txlog'),
background=True,
stdout=open(self.txlog_file, 'w'))
self.txlog = framework.Tailer(open(self.txlog_file),
flush=self.tablet.flush)
self.log = framework.Tailer(open(
os.path.join(environment.vtlogroot, 'vttablet.INFO')),
flush=self.tablet.flush)
break
except dbexceptions.OperationalError:
if i == 29:
raise
time.sleep(1)
self.postSetup()
def tearDown(self):
self.preTeardown()
self.tablet.kill_vttablet()
try:
utils.wait_procs([self.tablet.teardown_mysql()])
except:
# FIXME: remove
pass
if getattr(self, "txlogger", None):
self.txlogger.terminate()
environment.topo_server_teardown()
utils.kill_sub_processes()
def mysql_connect(self, dbname=''):
return self.tablet.connect()
