def setUpModule():
global vtgateclienttest_process
global vtgateclienttest_port
global vtgateclienttest_grpc_port
try:
environment.topo_server().setup()
vtgateclienttest_port = environment.reserve_ports(1)
args = environment.binary_args('vtgateclienttest') + [
'-log_dir', environment.vtlogroot,
'-port', str(vtgateclienttest_port),
]
if protocols_flavor().vtgate_python_protocol() == 'grpc':
vtgateclienttest_grpc_port = environment.reserve_ports(1)
args.extend(['-grpc_port', str(vtgateclienttest_grpc_port)])
if protocols_flavor().service_map():
args.extend(['-service_map', ','.join(protocols_flavor().service_map())])
vtgateclienttest_process = utils.run_bg(args)
utils.wait_for_vars('vtgateclienttest', vtgateclienttest_port)
except:
tearDownModule()
raise
def tearDownModule():
if utils.options.skip_teardown:
return
teardown_procs = [
shard_0_master.teardown_mysql(),
shard_0_replica1.teardown_mysql(),
shard_0_replica2.teardown_mysql(),
shard_0_rdonly.teardown_mysql(),
shard_0_backup.teardown_mysql(),
shard_1_master.teardown_mysql(),
shard_1_replica1.teardown_mysql(),
shard_2_master.teardown_mysql(),
shard_2_replica1.teardown_mysql(),
]
utils.wait_procs(teardown_procs, raise_on_error=False)
environment.topo_server().teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
shard_0_master.remove_tree()
shard_0_replica1.remove_tree()
shard_0_replica2.remove_tree()
shard_0_rdonly.remove_tree()
shard_0_backup.remove_tree()
shard_1_master.remove_tree()
shard_1_replica1.remove_tree()
shard_2_master.remove_tree()
shard_2_replica1.remove_tree()
def setUpModule():
try:
environment.topo_server().setup()
setup_procs = [
shard_master.init_mysql(),
shard_replica.init_mysql(),
shard_rdonly1.init_mysql(),
shard_0_master.init_mysql(),
shard_0_replica.init_mysql(),
shard_0_rdonly1.init_mysql(),
shard_1_master.init_mysql(),
shard_1_replica.init_mysql(),
shard_1_rdonly1.init_mysql(),
]
utils.wait_procs(setup_procs)
init_keyspace()
logging.debug('environment set up with the following shards and tablets:')
logging.debug('=========================================================')
logging.debug('TABLETS: test_keyspace/0:\n%s', all_shard_tablets)
logging.debug('TABLETS: test_keyspace/-80:\n%s', shard_0_tablets)
logging.debug('TABLETS: test_keyspace/80-:\n%s', shard_1_tablets)
except:
tearDownModule()
raise
def set_up(self):
try:
environment.topo_server().setup()
utils.wait_procs([t.init_mysql() for t in self.tablets])
utils.run_vtctl(['CreateKeyspace', self.keyspace])
utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', self.keyspace, 'keyspace_id', 'uint64'])
for t in self.tablets:
t.init_tablet(t.type, keyspace=self.keyspace, shard=t.shard)
utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace], auto_log=True)
for t in self.tablets:
t.create_db('vt_' + self.keyspace)
t.start_vttablet(
wait_for_state=None,
extra_args=['-queryserver-config-schema-reload-time', '1'],
)
for t in self.tablets:
t.wait_for_vttablet_state('SERVING')
for t in self.tablets:
if t.type == "master":
utils.run_vtctl(['ReparentShard', '-force', self.keyspace+'/'+t.shard, t.tablet_alias], auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace], auto_log=True)
if self.schema:
utils.run_vtctl(['ApplySchemaKeyspace', '-simple', '-sql', self.schema, self.keyspace])
if self.vschema:
if self.vschema[0] == '{':
utils.run_vtctl(['ApplyVSchema', "-vschema", self.vschema])
else:
utils.run_vtctl(['ApplyVSchema', "-vschema_file", self.vschema])
self.vtgate_server, self.vtgate_port = utils.vtgate_start(cache_ttl='500s', vtport=self.vtgate_port)
vtgate_client = zkocc.ZkOccConnection("localhost:%u" % self.vtgate_port, "test_nj", 30.0)
topology.read_topology(vtgate_client)
except:
self.shutdown()
raise
def setUp(self):
utils.wait_procs([self.tablet.init_mysql()])
self.tablet.mquery(
'', ['create database vt_test_keyspace', 'set global read_only = off'])
self.mysql_conn, mcu = self.tablet.connect('vt_test_keyspace')
with open(
os.path.join(self.vttop, 'test', 'test_data', 'test_schema.sql')) as f:
self.clean_sqls = []
self.init_sqls = []
clean_mode = False
for line in f:
line = line.rstrip()
if line == '# clean':
clean_mode = True
if line=='' or line.startswith('#'):
continue
if clean_mode:
self.clean_sqls.append(line)
else:
self.init_sqls.append(line)
try:
for line in self.init_sqls:
mcu.execute(line, {})
finally:
mcu.close()
customrules = os.path.join(environment.tmproot, 'customrules.json')
schema_override = os.path.join(environment.tmproot, 'schema_override.json')
self.create_schema_override(schema_override)
table_acl_config = os.path.join(
environment.vttop, 'test', 'test_data', 'table_acl_config.json')
environment.topo_server().setup()
self.create_customrules(customrules);
utils.run_vtctl('CreateKeyspace -force test_keyspace')
self.tablet.init_tablet('master', 'test_keyspace', '0')
if environment.topo_server().flavor() == 'zookeeper':
self.tablet.start_vttablet(
memcache=self.memcache,
zkcustomrules='/zk/test_ca/config/customrules/testrules',
schema_override=schema_override,
table_acl_config=table_acl_config,
)
else:
self.tablet.start_vttablet(
memcache=self.memcache,
filecustomrules=customrules,
schema_override=schema_override,
table_acl_config=table_acl_config,
)
self.conn = self.connect()
self.txlogger = utils.curl(
self.url('/debug/txlog'), background=True,
stdout=open(self.txlog_file, 'w'))
self.txlog = framework.Tailer(self.txlog_file, flush=self.tablet.flush)
self.log = framework.Tailer(
os.path.join(environment.vtlogroot, 'vttablet.INFO'),
flush=self.tablet.flush)
self.querylog = Querylog(self)
def setUpModule():
try:
environment.topo_server().setup()
_init_mysql(all_tablets)
utils.run_vtctl(['CreateKeyspace', test_keyspace])
shard_0_master.init_tablet('replica', test_keyspace, '0')
shard_0_replica1.init_tablet('replica', test_keyspace, '0')
shard_0_replica2.init_tablet('replica', test_keyspace, '0')
shard_0_rdonly.init_tablet('rdonly', test_keyspace, '0')
shard_0_backup.init_tablet('backup', test_keyspace, '0')
shard_1_master.init_tablet('replica', test_keyspace, '1')
shard_1_replica1.init_tablet('replica', test_keyspace, '1')
utils.Vtctld().start(enable_schema_change_dir=True)
# create databases, start the tablets
for t in initial_tablets:
t.create_db(db_name)
t.start_vttablet(wait_for_state=None)
# wait for the tablets to start
for t in initial_tablets:
t.wait_for_vttablet_state('NOT_SERVING')
except Exception as setup_exception: # pylint: disable=broad-except
try:
tearDownModule()
except Exception as e: # pylint: disable=broad-except
logging.exception('Tearing down a failed setUpModule() failed: %s', e)
raise setup_exception
def tearDownModule():
global vtgate_server
logging.debug("in tearDownModule")
if utils.options.skip_teardown:
return
logging.debug("Tearing down the servers and setup")
utils.vtgate_kill(vtgate_server)
tablet.kill_tablets([shard_0_master, shard_0_replica, shard_1_master,
shard_1_replica])
teardown_procs = [shard_0_master.teardown_mysql(),
shard_0_replica.teardown_mysql(),
shard_1_master.teardown_mysql(),
shard_1_replica.teardown_mysql(),
]
utils.wait_procs(teardown_procs, raise_on_error=False)
environment.topo_server().teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
shard_0_master.remove_tree()
shard_0_replica.remove_tree()
shard_1_master.remove_tree()
shard_1_replica.remove_tree()
def setUpModule():
try:
environment.topo_server().setup()
setup_procs = [
shard_0_master.init_mysql(),
shard_0_replica.init_mysql(),
shard_0_ny_rdonly.init_mysql(),
shard_1_master.init_mysql(),
shard_1_slave1.init_mysql(),
shard_1_slave2.init_mysql(),
shard_1_ny_rdonly.init_mysql(),
shard_1_rdonly1.init_mysql(),
shard_2_master.init_mysql(),
shard_2_replica1.init_mysql(),
shard_2_replica2.init_mysql(),
shard_3_master.init_mysql(),
shard_3_replica.init_mysql(),
shard_3_rdonly1.init_mysql(),
]
utils.Vtctld().start()
utils.wait_procs(setup_procs)
except:
tearDownModule()
raise
def setUpModule():
try:
environment.topo_server().setup()
except:
tearDownModule()
raise
def tearDownModule():
utils.required_teardown()
if utils.options.skip_teardown:
return
if use_mysqlctld:
# Try to terminate mysqlctld gracefully, so it kills its mysqld.
for proc in setup_procs:
utils.kill_sub_process(proc, soft=True)
teardown_procs = setup_procs
else:
teardown_procs = [
tablet_master.teardown_mysql(),
tablet_replica1.teardown_mysql(),
tablet_replica2.teardown_mysql(),
]
utils.wait_procs(teardown_procs, raise_on_error=False)
environment.topo_server().teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
tablet_master.remove_tree()
tablet_replica1.remove_tree()
tablet_replica2.remove_tree()
def tearDownModule():
if utils.options.skip_teardown:
return
tablet.kill_tablets([src_master, src_replica, src_rdonly1, src_rdonly2,
dst_master, dst_replica])
teardown_procs = [
src_master.teardown_mysql(),
src_replica.teardown_mysql(),
src_rdonly1.teardown_mysql(),
src_rdonly2.teardown_mysql(),
dst_master.teardown_mysql(),
dst_replica.teardown_mysql(),
]
utils.wait_procs(teardown_procs, raise_on_error=False)
environment.topo_server().teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
src_master.remove_tree()
src_replica.remove_tree()
src_rdonly1.remove_tree()
src_rdonly2.remove_tree()
dst_master.remove_tree()
dst_replica.remove_tree()
def tearDownModule():
utils.required_teardown()
if utils.options.skip_teardown:
return
if utils.vtgate:
utils.vtgate.kill()
teardown_procs = [
source_master.teardown_mysql(),
source_replica.teardown_mysql(),
source_rdonly1.teardown_mysql(),
source_rdonly2.teardown_mysql(),
destination_master.teardown_mysql(),
destination_replica.teardown_mysql(),
destination_rdonly1.teardown_mysql(),
destination_rdonly2.teardown_mysql(),
]
utils.wait_procs(teardown_procs, raise_on_error=False)
environment.topo_server().teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
source_master.remove_tree()
source_replica.remove_tree()
source_rdonly1.remove_tree()
source_rdonly2.remove_tree()
destination_master.remove_tree()
destination_replica.remove_tree()
destination_rdonly1.remove_tree()
destination_rdonly2.remove_tree()
def tearDown(self):
tablet.Tablet.check_vttablet_count()
environment.topo_server().wipe()
for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
t.reset_replication()
t.clean_dbs()
super(TestReparent, self).tearDown()
def tearDownModule():
if utils.options.skip_teardown:
return
tablet.kill_tablets([shard_0_master, shard_0_replica,
shard_1_master, shard_1_replica])
teardown_procs = [
shard_0_master.teardown_mysql(),
shard_0_replica.teardown_mysql(),
shard_1_master.teardown_mysql(),
shard_1_replica.teardown_mysql(),
unsharded_master.teardown_mysql(),
unsharded_replica.teardown_mysql(),
]
utils.wait_procs(teardown_procs, raise_on_error=False)
environment.topo_server().teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
shard_0_master.remove_tree()
shard_0_replica.remove_tree()
shard_1_master.remove_tree()
shard_1_replica.remove_tree()
unsharded_master.remove_tree()
unsharded_replica.remove_tree()
def tearDownModule():
if utils.options.skip_teardown:
return
if use_mysqlctld:
# Try to terminate mysqlctld gracefully, so it kills its mysqld.
for proc in setup_procs:
utils.kill_sub_process(proc, soft=True)
teardown_procs = setup_procs
else:
teardown_procs = [
tablet_62344.teardown_mysql(),
tablet_31981.teardown_mysql(),
]
utils.wait_procs(teardown_procs, raise_on_error=False)
environment.topo_server().teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
tablet_62344.remove_tree()
tablet_31981.remove_tree()
path = os.path.join(environment.vtdataroot, 'snapshot')
try:
shutil.rmtree(path)
except OSError as e:
logging.debug("removing snapshot %s: %s", path, str(e))
def tearDown(self):
tablet.Tablet.check_vttablet_count()
environment.topo_server().wipe()
for t in [tablet_62344, tablet_62044]:
t.reset_replication()
t.set_semi_sync_enabled(master=False)
t.clean_dbs()
def tearDownModule():
if utils.options.skip_teardown:
return
environment.topo_server().teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
def setUpModule():
try:
environment.topo_server().setup()
setup_procs = [t.init_mysql() for t in all_tablets]
utils.wait_procs(setup_procs)
except:
tearDownModule()
raise
def zkocc_start(cells=['test_nj'], extra_params=[]):
args = environment.binary_args('zkocc') + [
'-port', str(environment.topo_server().zkocc_port_base),
'-stderrthreshold=ERROR',
] + extra_params + cells
sp = run_bg(args)
wait_for_vars("zkocc", environment.topo_server().zkocc_port_base)
return sp
def tearDown(self):
tablet.Tablet.check_vttablet_count()
environment.topo_server().wipe()
for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
t.reset_replication()
t.set_semi_sync_enabled(master=False)
t.clean_dbs(include_vt=True)
super(TestReparent, self).tearDown()
def setUp(self):
environment.topo_server().wipe()
self.vtgate_zk, self.vtgate_zk_port = utils.vtgate_start()
if environment.topo_server().flavor() == 'zookeeper':
self.zkocc_server = utils.zkocc_start()
self.vtgate_zkocc, self.vtgate_zkocc_port = utils.vtgate_start(topo_impl="zkocc")
self.topo = zkocc.ZkOccConnection("localhost:%u" % environment.topo_server().zkocc_port_base, 'test_nj', 30)
self.topo.dial()
def test_actions_and_timeouts(self):
# Start up a master mysql and vttablet
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
tablet_62344.init_tablet('master', 'test_keyspace', '0')
utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
utils.validate_topology()
srvShard = utils.run_vtctl_json(['GetSrvShard', 'test_nj',
'test_keyspace/0'])
self.assertEqual(srvShard['MasterCell'], 'test_nj')
tablet_62344.create_db('vt_test_keyspace')
tablet_62344.start_vttablet()
utils.run_vtctl(['Ping', tablet_62344.tablet_alias])
# schedule long action in the background, sleep a little bit to make sure
# it started to run
args = (environment.binary_args('vtctl') +
environment.topo_server().flags() +
['-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol(),
'-tablet_protocol', protocols_flavor().tabletconn_protocol(),
'-log_dir', environment.vtlogroot,
'Sleep', tablet_62344.tablet_alias, '10s'])
bg = utils.run_bg(args)
time.sleep(3)
# try a frontend RefreshState that should timeout as the tablet is busy
# running the other one
stdout, stderr = utils.run_vtctl(['-wait-time', '3s',
'RefreshState', tablet_62344.tablet_alias],
expect_fail=True)
self.assertIn(protocols_flavor().rpc_timeout_message(), stderr)
# wait for the background vtctl
bg.wait()
if environment.topo_server().flavor() == 'zookeeper':
# extra small test: we ran for a while, get the states we were in,
# make sure they're accounted for properly
# first the query engine States
v = utils.get_vars(tablet_62344.port)
logging.debug("vars: %s" % str(v))
# then the Zookeeper connections
if v['ZkMetaConn']['test_nj']['Current'] != 'Connected':
self.fail('invalid zk test_nj state: %s' %
v['ZkMetaConn']['test_nj']['Current'])
if v['ZkMetaConn']['global']['Current'] != 'Connected':
self.fail('invalid zk global state: %s' %
v['ZkMetaConn']['global']['Current'])
if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9:
self.fail('not enough time in Connected state: %u',
v['ZkMetaConn']['test_nj']['DurationConnected'])
if v['TabletType'] != 'master':
self.fail('TabletType not exported correctly')
tablet_62344.kill_vttablet()
def test_actions_and_timeouts(self):
# Start up a master mysql and vttablet
utils.run_vtctl(["CreateKeyspace", "test_keyspace"])
tablet_62344.init_tablet("master", "test_keyspace", "0")
utils.run_vtctl(["RebuildShardGraph", "test_keyspace/0"])
utils.validate_topology()
self._check_srv_shard()
tablet_62344.create_db("vt_test_keyspace")
tablet_62344.start_vttablet()
utils.run_vtctl(["Ping", tablet_62344.tablet_alias])
# schedule long action in the background, sleep a little bit to make sure
# it started to run
args = (
environment.binary_args("vtctl")
+ environment.topo_server().flags()
+ [
"-tablet_manager_protocol",
protocols_flavor().tablet_manager_protocol(),
"-tablet_protocol",
protocols_flavor().tabletconn_protocol(),
"-log_dir",
environment.vtlogroot,
"Sleep",
tablet_62344.tablet_alias,
"10s",
]
)
bg = utils.run_bg(args)
time.sleep(3)
# try a frontend RefreshState that should timeout as the tablet is busy
# running the other one
_, stderr = utils.run_vtctl(["-wait-time", "3s", "RefreshState", tablet_62344.tablet_alias], expect_fail=True)
self.assertIn(protocols_flavor().rpc_timeout_message(), stderr)
# wait for the background vtctl
bg.wait()
if environment.topo_server().flavor() == "zookeeper":
# extra small test: we ran for a while, get the states we were in,
# make sure they're accounted for properly
# first the query engine States
v = utils.get_vars(tablet_62344.port)
logging.debug("vars: %s", v)
# then the Zookeeper connections
if v["ZkCachedConn"]["test_nj"] != "Connected":
self.fail("invalid zk test_nj state: %s" % v["ZkCachedConn"]["test_nj"])
if v["ZkCachedConn"]["global"] != "Connected":
self.fail("invalid zk global state: %s" % v["ZkCachedConn"]["global"])
if v["TabletType"] != "master":
self.fail("TabletType not exported correctly")
tablet_62344.kill_vttablet()
def shutdown(self):
tablet.kill_tablets(self.tablets)
teardown_procs = [t.teardown_mysql() for t in self.tablets]
utils.wait_procs(teardown_procs, raise_on_error=False)
environment.topo_server().teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
for t in self.tablets:
t.remove_tree()
def setUp(self):
utils.wait_procs([self.tablet.init_mysql()])
self.tablet.mquery("", ["create database vt_test_keyspace", "set global read_only = off"])
self.mysql_conn, mcu = self.tablet.connect('vt_test_keyspace')
with open(os.path.join(self.vttop, "test", "test_data", "test_schema.sql")) as f:
self.clean_sqls = []
self.init_sqls = []
clean_mode = False
for line in f:
line = line.rstrip()
if line == "# clean":
clean_mode = True
if line=='' or line.startswith("#"):
continue
if clean_mode:
self.clean_sqls.append(line)
else:
self.init_sqls.append(line)
try:
for line in self.init_sqls:
mcu.execute(line, {})
finally:
mcu.close()
customrules = os.path.join(environment.tmproot, 'customrules.json')
self.create_customrules(customrules)
schema_override = os.path.join(environment.tmproot, 'schema_override.json')
self.create_schema_override(schema_override)
table_acl_config = os.path.join(environment.vttop, 'test', 'test_data', 'table_acl_config.json')
if self.env == 'vttablet':
environment.topo_server().setup()
utils.run_vtctl('CreateKeyspace -force test_keyspace')
self.tablet.init_tablet('master', 'test_keyspace', '0')
self.tablet.start_vttablet(
memcache=self.memcache,
customrules=customrules,
schema_override=schema_override,
table_acl_config=table_acl_config,
auth=True,
)
else:
self.tablet.start_vtocc(
memcache=self.memcache,
customrules=customrules,
schema_override=schema_override,
table_acl_config=table_acl_config,
auth=True,
keyspace="test_keyspace", shard="0",
)
self.conn = self.connect()
self.txlogger = utils.curl(self.url('/debug/txlog'), background=True, stdout=open(self.txlog_file, 'w'))
self.txlog = framework.Tailer(self.txlog_file, flush=self.tablet.flush)
self.log = framework.Tailer(os.path.join(environment.vtlogroot, '%s.INFO' % self.env), flush=self.tablet.flush)
self.querylog = Querylog(self)
def setUpModule():
global vtgate_server
global vtgate_port
global vtgate_socket_file
global master_start_position
try:
environment.topo_server().setup()
# start mysql instance external to the test
setup_procs = [master_tablet.init_mysql(),
replica_tablet.init_mysql()]
utils.wait_procs(setup_procs)
# Start up a master mysql and vttablet
logging.debug('Setting up tablets')
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
master_tablet.init_tablet('master', 'test_keyspace', '0')
replica_tablet.init_tablet('replica', 'test_keyspace', '0')
utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
utils.validate_topology()
master_tablet.create_db('vt_test_keyspace')
master_tablet.create_db('other_database')
replica_tablet.create_db('vt_test_keyspace')
replica_tablet.create_db('other_database')
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])
vtgate_socket_file = environment.tmproot + '/vtgate.sock'
vtgate_server, vtgate_port = utils.vtgate_start(
socket_file=vtgate_socket_file)
master_tablet.start_vttablet()
replica_tablet.start_vttablet()
utils.run_vtctl(['SetReadWrite', master_tablet.tablet_alias])
utils.check_db_read_write(master_tablet.tablet_uid)
for t in [master_tablet, replica_tablet]:
t.reset_replication()
utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/0',
master_tablet.tablet_alias], auto_log=True)
# reset counter so tests don't assert
tablet.Tablet.tablets_running = 0
master_start_position = _get_master_current_position()
master_tablet.mquery('vt_test_keyspace', _create_vt_insert_test)
master_tablet.mquery('vt_test_keyspace', _create_vt_a)
master_tablet.mquery('vt_test_keyspace', _create_vt_b)
utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias])
utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias])
except:
tearDownModule()
raise
def setUpModule():
try:
environment.topo_server().setup()
setup_procs = [t.init_mysql(use_rbr=base_sharding.use_rbr)
for t in all_tablets]
utils.Vtctld().start()
utils.wait_procs(setup_procs)
except:
tearDownModule()
raise
def setUpModule():
global keyspace_env
global shard_0_master
global shard_1_master
global lookup_master
logging.debug('in setUpModule')
try:
environment.topo_server().setup()
logging.debug('Setting up tablets')
keyspace_env = keyspace_util.TestEnv()
keyspace_env.launch(
'user',
shards=['-80', '80-'],
ddls=[
create_vt_user,
create_vt_user2,
create_vt_user_extra,
create_vt_music,
create_vt_music_extra,
create_join_user,
create_join_user_extra,
create_join_name_info,
create_twopc_user,
],
rdonly_count=1, # to test SplitQuery
twopc_coordinator_address='localhost:15028', # enables 2pc
)
keyspace_env.launch(
'lookup',
ddls=[
create_vt_user_seq,
create_vt_music_seq,
create_vt_main_seq,
create_music_user_map,
create_name_user2_map,
create_main,
create_twopc_lookup,
],
twopc_coordinator_address='localhost:15028', # enables 2pc
)
shard_0_master = keyspace_env.tablet_map['user.-80.master']
shard_1_master = keyspace_env.tablet_map['user.80-.master']
lookup_master = keyspace_env.tablet_map['lookup.0.master']
utils.apply_vschema(vschema)
utils.VtGate().start(
tablets=[shard_0_master, shard_1_master, lookup_master],
extra_args=['-transaction_mode', 'TWOPC'])
utils.vtgate.wait_for_endpoints('user.-80.master', 1)
utils.vtgate.wait_for_endpoints('user.80-.master', 1)
utils.vtgate.wait_for_endpoints('lookup.0.master', 1)
except:
tearDownModule()
raise
def setUpModule():
try:
environment.topo_server().setup()
_init_mysql(tablets)
utils.run_vtctl(['CreateKeyspace', test_keyspace])
shard_0_master.init_tablet( 'master', test_keyspace, '0')
shard_0_replica1.init_tablet('replica', test_keyspace, '0')
shard_0_replica2.init_tablet('replica', test_keyspace, '0')
shard_0_rdonly.init_tablet( 'rdonly', test_keyspace, '0')
shard_0_backup.init_tablet( 'backup', test_keyspace, '0')
shard_1_master.init_tablet( 'master', test_keyspace, '1')
shard_1_replica1.init_tablet('replica', test_keyspace, '1')
utils.run_vtctl(['RebuildKeyspaceGraph', test_keyspace], auto_log=True)
# run checks now before we start the tablets
utils.validate_topology()
utils.Vtctld().start()
# create databases, start the tablets
for t in tablets:
t.create_db(db_name)
t.start_vttablet(wait_for_state=None)
# wait for the tablets to start
shard_0_master.wait_for_vttablet_state('SERVING')
shard_0_replica1.wait_for_vttablet_state('SERVING')
shard_0_replica2.wait_for_vttablet_state('SERVING')
shard_0_rdonly.wait_for_vttablet_state('SERVING')
shard_0_backup.wait_for_vttablet_state('NOT_SERVING')
shard_1_master.wait_for_vttablet_state('SERVING')
shard_1_replica1.wait_for_vttablet_state('SERVING')
# make sure all replication is good
for t in tablets:
t.reset_replication()
utils.run_vtctl(['InitShardMaster', test_keyspace+'/0',
shard_0_master.tablet_alias], auto_log=True)
utils.run_vtctl(['InitShardMaster', test_keyspace+'/1',
shard_1_master.tablet_alias], auto_log=True)
utils.run_vtctl(['ValidateKeyspace', '-ping-tablets', test_keyspace])
# check after all tablets are here and replication is fixed
utils.validate_topology(ping_tablets=True)
except Exception as setup_exception:
try:
tearDownModule()
except Exception as e:
logging.exception("Tearing down a failed setUpModule() failed: %s", e)
raise setup_exception
def tearDownModule():
logging.debug('in tearDownModule')
if utils.options.skip_teardown:
return
logging.debug('Tearing down the servers and setup')
keyspace_env.teardown()
environment.topo_server().teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
def setUpModule():
logging.debug('in setUpModule')
try:
environment.topo_server().setup()
# start mysql instance external to the test
setup_procs = [
shard_0_master.init_mysql(),
shard_0_replica1.init_mysql(),
]
utils.wait_procs(setup_procs)
setup_tablets()
setup_vtgate()
# After VTGate comes up, populate it with some initial data
initial_writes(0, keyrange.KeyRange(''))
except Exception, e:
logging.exception('error during set up: %s', e)
tearDownModule()
raise
def tearDownModule():
if utils.options.skip_teardown:
return
shard_0_master.kill_vttablet()
shard_0_slave.kill_vttablet()
teardown_procs = [
shard_0_master.teardown_mysql(),
shard_0_slave.teardown_mysql(),
]
utils.wait_procs(teardown_procs, raise_on_error=False)
environment.topo_server().teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
shard_0_master.remove_tree()
shard_0_slave.remove_tree()
def tearDownModule():
utils.required_teardown()
if utils.options.skip_teardown:
return
tablet.kill_tablets([master_tablet, replica_tablet])
teardown_procs = [
master_tablet.teardown_mysql(),
replica_tablet.teardown_mysql(),
]
utils.wait_procs(teardown_procs, raise_on_error=False)
environment.topo_server().teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
master_tablet.remove_tree()
replica_tablet.remove_tree()
def setUpModule():
try:
environment.topo_server().setup()
setup_procs = [
shard_master.init_mysql(),
shard_replica.init_mysql(),
shard_rdonly1.init_mysql(),
shard_0_master.init_mysql(),
shard_0_replica.init_mysql(),
shard_0_rdonly1.init_mysql(),
shard_1_master.init_mysql(),
shard_1_replica.init_mysql(),
shard_1_rdonly1.init_mysql(),
]
utils.wait_procs(setup_procs)
except:
tearDownModule()
raise
def tearDownModule():
global __tablets
if utils.options.skip_teardown:
return
if __tablets is not None:
tablet.kill_tablets(__tablets)
teardown_procs = []
for t in __tablets:
teardown_procs.append(t.teardown_mysql())
utils.wait_procs(teardown_procs, raise_on_error=False)
environment.topo_server().teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
if __tablets is not None:
for t in __tablets:
t.remove_tree()
def setUpModule():
global vtgate_server, vtgate_port
logging.debug("in setUpModule")
try:
environment.topo_server().setup()
setup_topology()
# start mysql instance external to the test
global __tablets
setup_procs = []
for tablet in __tablets:
setup_procs.append(tablet.init_mysql())
utils.wait_procs(setup_procs)
create_db()
start_tablets()
vtgate_server, vtgate_port = utils.vtgate_start()
except:
tearDownModule()
raise
def setUpModule():
try:
environment.topo_server().setup()
setup_procs = [
source_master.init_mysql(),
source_replica.init_mysql(),
source_rdonly1.init_mysql(),
source_rdonly2.init_mysql(),
destination_master.init_mysql(),
destination_replica.init_mysql(),
destination_rdonly1.init_mysql(),
destination_rdonly2.init_mysql(),
]
utils.Vtctld().start()
utils.wait_procs(setup_procs)
except:
tearDownModule()
raise
def setUpModule():
try:
if environment.topo_server().flavor() == 'zookeeper':
# this is a one-off test to make sure our zookeeper implementation
# behaves with a server that is not DNS-resolveable
environment.topo_server().setup(add_bad_host=True)
else:
environment.topo_server().setup()
# start mysql instance external to the test
setup_procs = [
tablet_62344.init_mysql(),
tablet_62044.init_mysql(),
]
utils.Vtctld().start()
utils.wait_procs(setup_procs)
except:
tearDownModule()
raise
def setUpModule():
global keyspace_env
global shard_0_master
global shard_1_master
global lookup_master
logging.debug('in setUpModule')
try:
environment.topo_server().setup()
logging.debug('Setting up tablets')
keyspace_env = keyspace_util.TestEnv()
keyspace_env.launch(
'user',
shards=['-80', '80-'],
ddls=[
create_vt_user,
create_vt_user2,
create_vt_user_extra,
create_vt_music,
create_vt_music_extra,
create_join_user,
create_join_user_extra,
create_join_name_info,
],
)
keyspace_env.launch(
'lookup',
ddls=[
create_vt_user_seq,
create_vt_music_seq,
create_music_user_map,
create_name_user2_map,
],
)
shard_0_master = keyspace_env.tablet_map['user.-80.master']
shard_1_master = keyspace_env.tablet_map['user.80-.master']
lookup_master = keyspace_env.tablet_map['lookup.0.master']
utils.apply_vschema(vschema)
utils.VtGate().start()
except:
tearDownModule()
raise
def setUpModule():
global vtgate_server
global vtgate_port
try:
environment.topo_server().setup()
# start mysql instance external to the test
setup_procs = [master_tablet.init_mysql(), replica_tablet.init_mysql()]
utils.wait_procs(setup_procs)
# Start up a master mysql and vttablet
logging.debug("Setting up tablets")
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
master_tablet.init_tablet('master', 'test_keyspace', '0')
replica_tablet.init_tablet('replica', 'test_keyspace', '0')
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])
utils.validate_topology()
master_tablet.populate('vt_test_keyspace', create_vt_insert_test)
replica_tablet.populate('vt_test_keyspace', create_vt_insert_test)
vtgate_server, vtgate_port = utils.vtgate_start()
master_tablet.start_vttablet(memcache=True, wait_for_state=None)
replica_tablet.start_vttablet(memcache=True, wait_for_state=None)
master_tablet.wait_for_vttablet_state('SERVING')
replica_tablet.wait_for_vttablet_state('SERVING')
utils.run_vtctl([
'ReparentShard', '-force', 'test_keyspace/0',
master_tablet.tablet_alias
],
auto_log=True)
utils.validate_topology()
# restart the replica tablet so the stats are reset
replica_tablet.kill_vttablet()
replica_tablet.start_vttablet(memcache=True)
except:
tearDownModule()
raise
def tearDownModule():
logging.debug('in tearDownModule')
utils.required_teardown()
if utils.options.skip_teardown:
return
logging.debug('Tearing down the servers and setup')
tablet.kill_tablets([shard_0_master, shard_0_replica1])
teardown_procs = [
shard_0_master.teardown_mysql(),
shard_0_replica1.teardown_mysql(),
]
utils.wait_procs(teardown_procs, raise_on_error=False)
environment.topo_server().teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
shard_0_master.remove_tree()
shard_0_replica1.remove_tree()
def tearDownModule():
if utils.options.skip_teardown:
return
teardown_procs = [
tablet_62344.teardown_mysql(),
tablet_62044.teardown_mysql(),
tablet_41983.teardown_mysql(),
tablet_31981.teardown_mysql(),
]
utils.wait_procs(teardown_procs, raise_on_error=False)
environment.topo_server().teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
tablet_62344.remove_tree()
tablet_62044.remove_tree()
tablet_41983.remove_tree()
tablet_31981.remove_tree()
def start(self, cell='test_nj', retry_count=2,
topo_impl=None, cache_ttl='1s',
extra_args=None, tablets=None,
tablet_types_to_wait='MASTER,REPLICA',
l2vtgates=None,
cells_to_watch=None):
"""Start vtgate. Saves it into the global vtgate variable if not set yet."""
args = environment.binary_args('vtgate') + [
'-port', str(self.port),
'-cell', cell,
'-retry-count', str(retry_count),
'-log_dir', environment.vtlogroot,
'-srv_topo_cache_ttl', cache_ttl,
'-srv_topo_cache_refresh', cache_ttl,
'-tablet_protocol', protocols_flavor().tabletconn_protocol(),
'-stderrthreshold', get_log_level(),
'-normalize_queries',
'-gateway_implementation', vtgate_gateway_flavor().flavor(),
]
if cells_to_watch:
args.extend(vtgate_gateway_flavor().flags(cell=cells_to_watch, tablets=tablets))
else:
args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets))
if l2vtgates:
args.extend(['-l2vtgate_addrs', ','.join(l2vtgates)])
if tablet_types_to_wait:
args.extend(['-tablet_types_to_wait', tablet_types_to_wait])
if protocols_flavor().vtgate_protocol() == 'grpc':
args.extend(['-grpc_port', str(self.grpc_port)])
args.extend(['-grpc_max_message_size',
str(environment.grpc_max_message_size)])
if protocols_flavor().service_map():
args.extend(['-service_map', ','.join(protocols_flavor().service_map())])
if topo_impl:
args.extend(['-topo_implementation', topo_impl])
else:
args.extend(environment.topo_server().flags())
if extra_args:
args.extend(extra_args)
if self.mysql_port:
args.extend(['-mysql_server_port', str(self.mysql_port)])
self.proc = run_bg(args)
# We use a longer timeout here, as we may be waiting for the initial
# state of a few tablets.
wait_for_vars('vtgate', self.port, timeout=20.0)
global vtgate
if not vtgate:
vtgate = self
def setUpModule():
try:
environment.topo_server().setup()
# start mysql instance external to the test
global setup_procs
setup_procs = [
tablet_master.init_mysql(),
tablet_replica1.init_mysql(),
tablet_replica2.init_mysql(),
]
if use_mysqlctld:
tablet_master.wait_for_mysqlctl_socket()
tablet_replica1.wait_for_mysqlctl_socket()
tablet_replica2.wait_for_mysqlctl_socket()
else:
utils.wait_procs(setup_procs)
except:
tearDownModule()
raise
def setUpModule():
global keyspace_env
global shard_0_master
global shard_1_master
global lookup_master
global vtgate_server
global vtgate_port
logging.debug("in setUpModule")
try:
environment.topo_server().setup()
logging.debug("Setting up tablets")
keyspace_env = keyspace_util.TestEnv()
keyspace_env.launch(
"user",
shards=["-80", "80-"],
ddls=[
create_vt_user,
create_vt_user2,
create_vt_user_extra,
create_vt_music,
create_vt_music_extra,
],
)
keyspace_env.launch(
"lookup",
ddls=[
create_vt_user_idx,
create_music_user_map,
create_name_user2_map,
],
)
shard_0_master = keyspace_env.tablet_map["user.-80.master"]
shard_1_master = keyspace_env.tablet_map["user.80-.master"]
lookup_master = keyspace_env.tablet_map["lookup.0.master"]
utils.apply_vschema(schema)
vtgate_server, vtgate_port = utils.vtgate_start()
except:
tearDownModule()
raise
def tearDownModule():
utils.required_teardown()
if utils.options.skip_teardown:
return
teardown_procs = [
shard_0_master.teardown_mysql(),
shard_0_replica.teardown_mysql(),
shard_0_ny_rdonly.teardown_mysql(),
shard_1_master.teardown_mysql(),
shard_1_slave1.teardown_mysql(),
shard_1_slave2.teardown_mysql(),
shard_1_ny_rdonly.teardown_mysql(),
shard_1_rdonly1.teardown_mysql(),
shard_2_master.teardown_mysql(),
shard_2_replica1.teardown_mysql(),
shard_2_replica2.teardown_mysql(),
shard_3_master.teardown_mysql(),
shard_3_replica.teardown_mysql(),
shard_3_rdonly1.teardown_mysql(),
]
utils.wait_procs(teardown_procs, raise_on_error=False)
environment.topo_server().teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
shard_0_master.remove_tree()
shard_0_replica.remove_tree()
shard_0_ny_rdonly.remove_tree()
shard_1_master.remove_tree()
shard_1_slave1.remove_tree()
shard_1_slave2.remove_tree()
shard_1_ny_rdonly.remove_tree()
shard_1_rdonly1.remove_tree()
shard_2_master.remove_tree()
shard_2_replica1.remove_tree()
shard_2_replica2.remove_tree()
shard_3_master.remove_tree()
shard_3_replica.remove_tree()
shard_3_rdonly1.remove_tree()
def setUp(self):
try:
os.makedirs(environment.tmproot)
except OSError:
# directory already exists
pass
try:
topo_flavor = environment.topo_server().flavor()
environment.topo_server().setup()
# start mysql instance external to the test
setup_procs = [
self.replica.init_mysql(),
self.master.init_mysql(),
]
utils.Vtctld().start()
logging.debug(utils.vtctld_connection)
utils.wait_procs(setup_procs)
for t in self.master, self.replica:
t.create_db('vt_test_keyspace')
self.master.init_tablet('replica',
'test_keyspace',
'0',
start=True)
self.replica.init_tablet('replica',
'test_keyspace',
'0',
start=True)
utils.run_vtctl([
'InitShardMaster', '-force', 'test_keyspace/0',
self.master.tablet_alias
])
self.master.mquery('vt_test_keyspace', _create_vt_insert_test)
for t in [self.master, self.replica]:
t.set_semi_sync_enabled(master=False, slave=False)
except Exception as e:
logging.exception(e)
self.tearDown()
def setUpModule():
try:
environment.topo_server().setup()
_init_mysql(all_tablets)
utils.run_vtctl(['CreateKeyspace', test_keyspace])
shard_0_master.init_tablet('master', test_keyspace, '0')
shard_0_replica1.init_tablet('replica', test_keyspace, '0')
shard_0_replica2.init_tablet('replica', test_keyspace, '0')
shard_0_rdonly.init_tablet('rdonly', test_keyspace, '0')
shard_0_backup.init_tablet('backup', test_keyspace, '0')
shard_1_master.init_tablet('master', test_keyspace, '1')
shard_1_replica1.init_tablet('replica', test_keyspace, '1')
# run checks now before we start the tablets
utils.validate_topology()
utils.Vtctld().start(enable_schema_change_dir=True)
# create databases, start the tablets
for t in initial_tablets:
t.create_db(db_name)
t.start_vttablet(wait_for_state=None)
# wait for the tablets to start
shard_0_master.wait_for_vttablet_state('SERVING')
shard_0_replica1.wait_for_vttablet_state('NOT_SERVING')
shard_0_replica2.wait_for_vttablet_state('NOT_SERVING')
shard_0_rdonly.wait_for_vttablet_state('NOT_SERVING')
shard_0_backup.wait_for_vttablet_state('NOT_SERVING')
shard_1_master.wait_for_vttablet_state('SERVING')
shard_1_replica1.wait_for_vttablet_state('NOT_SERVING')
except Exception as setup_exception: # pylint: disable=broad-except
try:
tearDownModule()
except Exception as e: # pylint: disable=broad-except
logging.exception('Tearing down a failed setUpModule() failed: %s',
e)
raise setup_exception
def setUpModule():
global keyspace_env
global shard_0_master
global shard_0_replica
global shard_1_master
global lookup_master
logging.debug('in setUpModule')
try:
environment.topo_server().setup()
logging.debug('Setting up tablets')
keyspace_env = keyspace_util.TestEnv()
keyspace_env.launch(
'user',
shards=['-80', '80-'],
ddls=[
create_sharded_message,
],
)
keyspace_env.launch(
'lookup',
ddls=[
create_unsharded_message,
],
)
shard_0_master = keyspace_env.tablet_map['user.-80.master']
shard_0_replica = keyspace_env.tablet_map['user.-80.replica.0']
shard_1_master = keyspace_env.tablet_map['user.80-.master']
lookup_master = keyspace_env.tablet_map['lookup.0.master']
utils.apply_vschema(vschema)
utils.VtGate().start(tablets=[
shard_0_master, shard_0_replica, shard_1_master, lookup_master
])
utils.vtgate.wait_for_endpoints('user.-80.master', 1)
utils.vtgate.wait_for_endpoints('user.-80.replica', 1)
utils.vtgate.wait_for_endpoints('user.80-.master', 1)
utils.vtgate.wait_for_endpoints('lookup.0.master', 1)
except:
tearDownModule()
raise
def setUpModule():
try:
environment.topo_server().setup()
utils.Vtctld().start()
setup_procs = [
master_tablet.init_mysql(),
replica_tablet.init_mysql(),
]
utils.wait_procs(setup_procs)
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
master_tablet.init_tablet('master', 'test_keyspace', '0')
replica_tablet.init_tablet('replica', 'test_keyspace', '0')
master_tablet.create_db('vt_test_keyspace')
replica_tablet.create_db('vt_test_keyspace')
except:
tearDownModule()
raise
def start(self):
args = environment.binary_args('vtctld') + [
'-debug',
'-web_dir',
environment.vttop + '/web/vtctld',
'--templates',
environment.vttop + '/go/cmd/vtctld/templates',
'--log_dir',
environment.vtlogroot,
'--port',
str(self.port),
'--schema_change_dir',
self.schema_change_dir,
'--schema_change_controller',
'local',
'--schema_change_check_interval',
'1',
'-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol(),
'-vtgate_protocol',
protocols_flavor().vtgate_protocol(),
'-tablet_protocol',
protocols_flavor().tabletconn_protocol(),
] + environment.topo_server().flags()
if protocols_flavor().service_map():
args.extend(
['-service_map', ','.join(protocols_flavor().service_map())])
if protocols_flavor().vtctl_client_protocol() == 'grpc':
args.extend(['-grpc_port', str(self.grpc_port)])
stdout_fd = open(os.path.join(environment.tmproot, 'vtctld.stdout'),
'w')
stderr_fd = open(os.path.join(environment.tmproot, 'vtctld.stderr'),
'w')
self.proc = run_bg(args, stdout=stdout_fd, stderr=stderr_fd)
# wait for the process to listen to RPC
timeout = 30
while True:
v = get_vars(self.port)
if v:
break
timeout = wait_step('waiting for vtctld to start',
timeout,
sleep_time=0.2)
# save the running instance so vtctl commands can be remote executed now
global vtctld, vtctld_connection
if not vtctld:
vtctld = self
protocol, endpoint = self.rpc_endpoint(python=True)
vtctld_connection = vtctl_client.connect(protocol, endpoint, 30)
return self.proc
def setUpModule():
global keyspace_env
global shard_0_master
try:
environment.topo_server().setup()
keyspace_env = keyspace_util.TestEnv()
keyspace_env.launch('user', ddls=[
create_vt_user,
create_main,
])
shard_0_master = keyspace_env.tablet_map['user.0.master']
utils.VtGate().start(
tablets=[shard_0_master],
extra_args=['-vschema_ddl_authorized_users', '%'],
)
utils.vtgate.wait_for_endpoints('user.0.master', 1)
except:
tearDownModule()
raise
def main():
parser = optparse.OptionParser(usage="usage: %prog [options]")
utils.add_options(parser)
(options, args) = parser.parse_args()
options.debug = True
utils.set_options(options)
env = keyspace_util.TestEnv()
vtgate_server = None
try:
environment.topo_server().setup()
env.launch(
"user",
shards=["-80", "80-"],
ddls=[
'create table user(user_id bigint, name varchar(128), primary key(user_id))',
'create table user_extra(user_id bigint, extra varchar(128), primary key(user_id))',
'create table music(user_id bigint, music_id bigint, primary key(user_id, music_id))',
'create table music_extra(music_id bigint, keyspace_id bigint unsigned, primary key(music_id))',
],
)
env.launch(
"lookup",
ddls=[
'create table user_idx(user_id bigint not null auto_increment, primary key(user_id))',
'create table name_user_idx(name varchar(128), user_id bigint, primary key(name, user_id))',
'create table music_user_idx(music_id bigint not null auto_increment, user_id bigint, primary key(music_id))',
],
)
utils.apply_vschema(vschema)
vtgate_server, vtgate_port = utils.vtgate_start(cache_ttl='500s')
utils.Vtctld().start()
print "vtgate:", vtgate_port
print "vtctld:", utils.vtctld.port
utils.pause("the cluster is up, press enter to shut it down...")
finally:
utils.vtgate_kill(vtgate_server)
env.teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
environment.topo_server().teardown()
def setUpModule():
try:
if utils.options.xvfb:
try:
# This will be killed automatically by utils.kill_sub_processes()
utils.run_bg(['Xvfb', ':15', '-ac'])
os.environ['DISPLAY'] = ':15'
except OSError as err:
# Despite running in background, utils.run_bg() will throw immediately
# if the Xvfb binary is not found.
logging.error(
"Can't start Xvfb (will try local DISPLAY instead): %s",
err)
environment.topo_server().setup()
setup_procs = [t.init_mysql() for t in tablets]
utils.Vtctld().start()
utils.wait_procs(setup_procs)
except:
tearDownModule()
raise
def setUpModule():
try:
environment.topo_server().setup()
# start mysql instance external to the test
setup_procs = [master_tablet.init_mysql(),
replica_tablet.init_mysql()]
utils.wait_procs(setup_procs)
# start a vtctld so the vtctl insert commands are just RPCs, not forks
utils.Vtctld().start()
# Start up a master mysql and vttablet
logging.debug('Setting up tablets')
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
master_tablet.init_tablet('master', 'test_keyspace', '0')
replica_tablet.init_tablet('replica', 'test_keyspace', '0')
utils.validate_topology()
master_tablet.populate('vt_test_keyspace', create_vt_insert_test)
replica_tablet.populate('vt_test_keyspace', create_vt_insert_test)
utils.VtGate().start()
master_tablet.start_vttablet(memcache=True, wait_for_state=None)
replica_tablet.start_vttablet(memcache=True, wait_for_state=None)
master_tablet.wait_for_vttablet_state('SERVING')
replica_tablet.wait_for_vttablet_state('SERVING')
utils.run_vtctl(['InitShardMaster', 'test_keyspace/0',
master_tablet.tablet_alias], auto_log=True)
utils.validate_topology()
# restart the replica tablet so the stats are reset
replica_tablet.kill_vttablet()
replica_tablet.start_vttablet(memcache=True)
except:
tearDownModule()
raise
def tearDownModule():
utils.required_teardown()
if utils.options.skip_teardown:
return
teardown_procs = [
tablet_master.teardown_mysql(
extra_args=['-db-credentials-file', db_credentials_file]),
tablet_replica1.teardown_mysql(
extra_args=['-db-credentials-file', db_credentials_file]),
tablet_replica2.teardown_mysql(
extra_args=['-db-credentials-file', db_credentials_file]),
]
utils.wait_procs(teardown_procs, raise_on_error=False)
environment.topo_server().teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
tablet_master.remove_tree()
tablet_replica1.remove_tree()
tablet_replica2.remove_tree()
def tearDownModule():
global vtgate_server
logging.debug("in tearDownModule")
if utils.options.skip_teardown:
return
logging.debug("Tearing down the servers and setup")
utils.vtgate_kill(vtgate_server)
tablet.kill_tablets([shard_0_master, shard_1_master, lookup_master])
teardown_procs = [shard_0_master.teardown_mysql(),
shard_1_master.teardown_mysql(),
lookup_master.teardown_mysql(),
]
utils.wait_procs(teardown_procs, raise_on_error=False)
environment.topo_server().teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
shard_0_master.remove_tree()
shard_1_master.remove_tree()
lookup_master.remove_tree()
def tearDownModule():
if utils.options.skip_teardown:
return
tablet.kill_tablets([src_master, src_replica, dst_master, dst_replica])
teardown_procs = [
src_master.teardown_mysql(),
src_replica.teardown_mysql(),
dst_master.teardown_mysql(),
dst_replica.teardown_mysql(),
]
utils.wait_procs(teardown_procs, raise_on_error=False)
environment.topo_server().teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
src_master.remove_tree()
src_replica.remove_tree()
dst_master.remove_tree()
dst_replica.remove_tree()
def setUp(self):
environment.topo_server().wipe()
if environment.topo_server().flavor() == 'zookeeper':
utils.run(environment.binary_argstr('zk')+' touch -p /zk/test_nj/vt/zkocc1')
utils.run(environment.binary_argstr('zk')+' touch -p /zk/test_nj/vt/zkocc2')
fd = tempfile.NamedTemporaryFile(dir=environment.tmproot, delete=False)
filename1 = fd.name
fd.write("Test data 1")
fd.close()
utils.run(environment.binary_argstr('zk')+' cp '+filename1+' /zk/test_nj/vt/zkocc1/data1')
fd = tempfile.NamedTemporaryFile(dir=environment.tmproot, delete=False)
filename2 = fd.name
fd.write("Test data 2")
fd.close()
utils.run(environment.binary_argstr('zk')+' cp '+filename2+' /zk/test_nj/vt/zkocc1/data2')
fd = tempfile.NamedTemporaryFile(dir=environment.tmproot, delete=False)
filename3 = fd.name
fd.write("Test data 3")
fd.close()
utils.run(environment.binary_argstr('zk')+' cp '+filename3+' /zk/test_nj/vt/zkocc1/data3')
def setUpModule():
try:
environment.topo_server().setup()
setup_procs = [
shard_master.init_mysql(),
shard_replica.init_mysql(),
shard_rdonly1.init_mysql(),
shard_0_master.init_mysql(),
shard_0_replica.init_mysql(),
shard_0_rdonly1.init_mysql(),
shard_1_master.init_mysql(),
shard_1_replica.init_mysql(),
shard_1_rdonly1.init_mysql(),
]
# we only use vtgate for testing SplitQuery works correctly.
# we want cache_ttl at zero so we re-read the topology for every test query.
utils.VtGate().start(cache_ttl='0')
utils.wait_procs(setup_procs)
except:
tearDownModule()
raise