def setUp(self):
utils.zk_wipe()
self.zkocc_server = utils.zkocc_start()
# the default topo implementation for vtgate is zookeeper
self.vtgate_zk, self.vtgate_zk_port = utils.vtgate_start()
self.vtgate_zkocc, self.vtgate_zkocc_port = utils.vtgate_start(topo_impl="zkocc")
self.topo = zkocc.ZkOccConnection("localhost:%u" % utils.zkocc_port_base, 'test_nj', 30)
self.topo.dial()
def setUp(self):
environment.topo_server_wipe()
self.vtgate_zk, self.vtgate_zk_port = utils.vtgate_start()
if environment.topo_server_implementation == 'zookeeper':
self.zkocc_server = utils.zkocc_start()
self.vtgate_zkocc, self.vtgate_zkocc_port = utils.vtgate_start(topo_impl="zkocc")
self.topo = zkocc.ZkOccConnection("localhost:%u" % environment.zkocc_port_base, 'test_nj', 30)
self.topo.dial()
def setup_tablets(): global vtgate_server global vtgate_port setup_sharded_keyspace() setup_unsharded_keyspace() vtgate_server, vtgate_port = utils.vtgate_start()
def setup_tablets(): global vtgate_server global vtgate_port setup_sharded_keyspace() setup_unsharded_keyspace() vtgate_server, vtgate_port = utils.vtgate_start()
def setup_tablets():
global vtgate_server
global vtgate_port
# Start up a master mysql and vttablet
logging.debug("Setting up tablets")
utils.run_vtctl(['CreateKeyspace', USER_KEYSACE])
utils.run_vtctl(['CreateKeyspace', LOOKUP_KEYSPACE])
utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', USER_KEYSACE,
'keyspace_id', 'uint64'])
shard_0_master.init_tablet('master', keyspace=USER_KEYSACE, shard='-80')
shard_1_master.init_tablet('master', keyspace=USER_KEYSACE, shard='80-')
lookup_master.init_tablet('master', keyspace=LOOKUP_KEYSPACE, shard='0')
for t in [shard_0_master, shard_1_master]:
t.create_db('vt_user_keyspace')
t.mquery('vt_user_keyspace', create_vt_user)
t.mquery('vt_user_keyspace', create_vt_user2)
t.mquery('vt_user_keyspace', create_vt_user_extra)
t.mquery('vt_user_keyspace', create_vt_music)
t.mquery('vt_user_keyspace', create_vt_music_extra)
t.start_vttablet(wait_for_state='SERVING')
utils.run_vtctl(['SetReadWrite', t.tablet_alias])
lookup_master.create_db('vt_lookup_keyspace')
lookup_master.mquery('vt_lookup_keyspace', create_vt_user_idx)
lookup_master.mquery('vt_lookup_keyspace', create_music_user_map)
lookup_master.mquery('vt_lookup_keyspace', create_name_user2_map)
lookup_master.start_vttablet(wait_for_state='SERVING')
utils.run_vtctl(['SetReadWrite', lookup_master.tablet_alias])
utils.run_vtctl(['RebuildKeyspaceGraph', USER_KEYSACE], auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', LOOKUP_KEYSPACE], auto_log=True)
vtgate_server, vtgate_port = utils.vtgate_start(schema=schema)
def test_vtgate_qps(self):
# create the topology
utils.run_vtctl('CreateKeyspace test_keyspace')
t = tablet.Tablet(tablet_uid=1, cell="nj")
t.init_tablet("master", "test_keyspace", "0")
t.update_addrs()
utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)
# start vtgate and the qps-er
vtgate_proc, vtgate_port = utils.vtgate_start()
qpser = utils.run_bg(environment.binary_path('zkclient2')+' -server localhost:%u -mode qps2 test_nj test_keyspace' % vtgate_port)
time.sleep(10)
utils.kill_sub_process(qpser)
# get the vtgate vars, make sure we have what we need
v = utils.get_vars(vtgate_port)
# some checks on performance / stats
# a typical workstation will do 38-40k QPS, check we have more than 10k
rpcCalls = v['TopoReaderRpcQueryCount']['test_nj']
if rpcCalls < 100000:
self.fail('QPS is too low: %u < 10000' % (rpcCalls / 10))
else:
logging.debug("Recorded qps: %u", rpcCalls / 10)
utils.vtgate_kill(vtgate_proc)
def setup_tablets():
global vtgate_server
global vtgate_port
# Start up a master mysql and vttablet
logging.debug("Setting up tablets")
utils.run_vtctl(['CreateKeyspace', USER_KEYSACE])
utils.run_vtctl(['CreateKeyspace', LOOKUP_KEYSPACE])
utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', USER_KEYSACE,
'keyspace_id', 'uint64'])
shard_0_master.init_tablet('master', keyspace=USER_KEYSACE, shard='-80')
shard_1_master.init_tablet('master', keyspace=USER_KEYSACE, shard='80-')
lookup_master.init_tablet('master', keyspace=LOOKUP_KEYSPACE, shard='0')
for t in [shard_0_master, shard_1_master]:
t.create_db('vt_user_keyspace')
t.mquery('vt_user_keyspace', create_vt_user)
t.mquery('vt_user_keyspace', create_vt_user2)
t.mquery('vt_user_keyspace', create_vt_user_extra)
t.mquery('vt_user_keyspace', create_vt_music)
t.mquery('vt_user_keyspace', create_vt_music_extra)
t.start_vttablet(wait_for_state='SERVING')
utils.run_vtctl(['SetReadWrite', t.tablet_alias])
lookup_master.create_db('vt_lookup_keyspace')
lookup_master.mquery('vt_lookup_keyspace', create_vt_user_idx)
lookup_master.mquery('vt_lookup_keyspace', create_music_user_map)
lookup_master.mquery('vt_lookup_keyspace', create_name_user2_map)
lookup_master.start_vttablet(wait_for_state='SERVING')
utils.run_vtctl(['SetReadWrite', lookup_master.tablet_alias])
utils.run_vtctl(['RebuildKeyspaceGraph', USER_KEYSACE], auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', LOOKUP_KEYSPACE], auto_log=True)
vtgate_server, vtgate_port = utils.vtgate_start(schema=schema)
def test_vtgate_qps(self):
# create the topology
utils.run_vtctl('CreateKeyspace test_keyspace')
t = tablet.Tablet(tablet_uid=1, cell="nj")
t.init_tablet("master", "test_keyspace", "0")
t.update_addrs()
utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)
# start vtgate and the qps-er
vtgate_proc, vtgate_port = utils.vtgate_start(
extra_args=['-cpu_profile', os.path.join(environment.tmproot,
'vtgate.pprof')])
qpser = utils.run_bg(environment.binary_args('zkclient2') + [
'-server', 'localhost:%u' % vtgate_port,
'-mode', 'qps',
'-zkclient_cpu_profile', os.path.join(environment.tmproot, 'zkclient2.pprof'),
'test_nj', 'test_keyspace'])
qpser.wait()
# get the vtgate vars, make sure we have what we need
v = utils.get_vars(vtgate_port)
# some checks on performance / stats
rpcCalls = v['TopoReaderRpcQueryCount']['test_nj']
if rpcCalls < MIN_QPS * 10:
self.fail('QPS is too low: %u < %u' % (rpcCalls / 10, MIN_QPS))
else:
logging.debug("Recorded qps: %u", rpcCalls / 10)
utils.vtgate_kill(vtgate_proc)
def test_vtgate_qps(self):
# create the topology
utils.run_vtctl('CreateKeyspace test_keyspace')
t = tablet.Tablet(tablet_uid=1, cell="nj")
t.init_tablet("master", "test_keyspace", "0")
t.update_addrs()
utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)
# start vtgate and the qps-er
vtgate_proc, vtgate_port = utils.vtgate_start(
extra_args=['-cpu_profile', os.path.join(environment.tmproot,
'vtgate.pprof')])
qpser = utils.run_bg(environment.binary_args('zkclient2') + [
'-server', 'localhost:%u' % vtgate_port,
'-mode', 'qps',
'-zkclient_cpu_profile', os.path.join(environment.tmproot, 'zkclient2.pprof'),
'test_nj', 'test_keyspace'])
qpser.wait()
# get the vtgate vars, make sure we have what we need
v = utils.get_vars(vtgate_port)
# some checks on performance / stats
rpcCalls = v['TopoReaderRpcQueryCount']['test_nj']
if rpcCalls < MIN_QPS * 10:
self.fail('QPS is too low: %u < %u' % (rpcCalls / 10, MIN_QPS))
else:
logging.debug("Recorded qps: %u", rpcCalls / 10)
utils.vtgate_kill(vtgate_proc)
def test_vtgate_qps(self):
# create the topology
utils.run_vtctl('CreateKeyspace test_keyspace')
t = tablet.Tablet(tablet_uid=1, cell="nj")
t.init_tablet("master", "test_keyspace", "0")
t.update_addrs()
utils.run_vtctl('RebuildShardGraph test_keyspace/0', auto_log=True)
utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)
# start vtgate and the qps-er
vtgate_proc, vtgate_port = utils.vtgate_start()
qpser = utils.run_bg(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode qps2 test_nj test_keyspace' % vtgate_port)
time.sleep(10)
utils.kill_sub_process(qpser)
# get the vtgate vars, make sure we have what we need
v = utils.get_vars(vtgate_port)
# some checks on performance / stats
# a typical workstation will do 38-40k QPS, check we have more than 15k
rpcCalls = v['TopoReaderRpcQueryCount']['test_nj']
if rpcCalls < 150000:
self.fail('QPS is too low: %u < 15000' % (rpcCalls / 10))
else:
logging.debug("Recorded qps: %u", rpcCalls / 10)
utils.vtgate_kill(vtgate_proc)
def set_up(self):
try:
environment.topo_server_setup()
utils.wait_procs([t.init_mysql() for t in self.tablets])
utils.run_vtctl(['CreateKeyspace', self.keyspace])
utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', self.keyspace, 'keyspace_id', 'uint64'])
for t in self.tablets:
t.init_tablet(t.type, keyspace=self.keyspace, shard=t.shard)
utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace], auto_log=True)
for t in self.tablets:
t.create_db('vt_' + self.keyspace)
t.mquery(t.dbname, create_table)
t.start_vttablet(wait_for_state=None)
for t in self.tablets:
t.wait_for_vttablet_state('SERVING')
for t in self.tablets:
if t.type == "master":
utils.run_vtctl(['ReparentShard', '-force', self.keyspace+'/'+t.shard, t.tablet_alias], auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace], auto_log=True)
self.vtgate_server, self.vtgate_port = utils.vtgate_start()
vtgate_client = zkocc.ZkOccConnection("localhost:%u" % self.vtgate_port, "test_nj", 30.0)
topology.read_topology(vtgate_client)
except:
self.shutdown()
raise
def set_up(self):
try:
environment.topo_server_setup()
utils.wait_procs([t.init_mysql() for t in self.tablets])
utils.run_vtctl(['CreateKeyspace', self.keyspace])
utils.run_vtctl([
'SetKeyspaceShardingInfo', '-force', self.keyspace,
'keyspace_id', 'uint64'
])
for t in self.tablets:
t.init_tablet(t.type, keyspace=self.keyspace, shard=t.shard)
utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace],
auto_log=True)
for t in self.tablets:
t.create_db('vt_' + self.keyspace)
t.mquery(t.dbname, create_table)
t.start_vttablet(wait_for_state=None)
for t in self.tablets:
t.wait_for_vttablet_state('SERVING')
for t in self.tablets:
if t.type == "master":
utils.run_vtctl([
'ReparentShard', '-force',
self.keyspace + '/' + t.shard, t.tablet_alias
],
auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace],
auto_log=True)
self.vtgate_server, self.vtgate_port = utils.vtgate_start()
vtgate_client = zkocc.ZkOccConnection(
"localhost:%u" % self.vtgate_port, "test_nj", 30.0)
topology.read_topology(vtgate_client)
except:
self.shutdown()
raise
def set_up(self):
try:
environment.topo_server().setup()
utils.wait_procs([t.init_mysql() for t in self.tablets])
utils.run_vtctl(['CreateKeyspace', self.keyspace])
utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', self.keyspace, 'keyspace_id', 'uint64'])
for t in self.tablets:
t.init_tablet(t.type, keyspace=self.keyspace, shard=t.shard)
utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace], auto_log=True)
for t in self.tablets:
t.create_db('vt_' + self.keyspace)
t.start_vttablet(
wait_for_state=None,
extra_args=['-queryserver-config-schema-reload-time', '1'],
)
for t in self.tablets:
t.wait_for_vttablet_state('SERVING')
for t in self.tablets:
if t.type == "master":
utils.run_vtctl(['ReparentShard', '-force', self.keyspace+'/'+t.shard, t.tablet_alias], auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace], auto_log=True)
if self.schema:
utils.run_vtctl(['ApplySchemaKeyspace', '-simple', '-sql', self.schema, self.keyspace])
if self.vschema:
if self.vschema[0] == '{':
utils.run_vtctl(['ApplyVSchema', "-vschema", self.vschema])
else:
utils.run_vtctl(['ApplyVSchema', "-vschema_file", self.vschema])
self.vtgate_server, self.vtgate_port = utils.vtgate_start(cache_ttl='500s', vtport=self.vtgate_port)
vtgate_client = zkocc.ZkOccConnection("localhost:%u" % self.vtgate_port, "test_nj", 30.0)
topology.read_topology(vtgate_client)
except:
self.shutdown()
raise
def setUpClass(klass):
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
utils.run_vtctl([
'CreateKeyspace', '--served-from',
'master:test_keyspace,replica:test_keyspace,rdonly:test_keyspace',
'redirected_keyspace'
])
shard_0_master.init_tablet('master', 'test_keyspace', '-80')
shard_0_replica.init_tablet('spare', 'test_keyspace', '-80')
shard_0_spare.init_tablet('spare', 'test_keyspace', '-80')
shard_1_master.init_tablet('master', 'test_keyspace', '80-')
shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')
idle.init_tablet('idle')
scrap.init_tablet('idle')
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'],
auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', 'redirected_keyspace'],
auto_log=True)
for t in [shard_0_master, shard_1_master, shard_1_replica]:
t.create_db('vt_test_keyspace')
t.start_vttablet(extra_args=vtctld.process_args())
shard_0_replica.create_db('vt_test_keyspace')
shard_0_replica.start_vttablet(extra_args=vtctld.process_args(),
target_tablet_type='replica',
wait_for_state='NOT_SERVING')
for t in scrap, idle, shard_0_spare:
t.start_vttablet(wait_for_state='NOT_SERVING',
extra_args=vtctld.process_args())
scrap.scrap()
for t in [
shard_0_master, shard_0_replica, shard_0_spare, shard_1_master,
shard_1_replica, idle, scrap
]:
t.reset_replication()
utils.run_vtctl([
'ReparentShard', '-force', 'test_keyspace/-80',
shard_0_master.tablet_alias
],
auto_log=True)
utils.run_vtctl([
'ReparentShard', '-force', 'test_keyspace/80-',
shard_1_master.tablet_alias
],
auto_log=True)
shard_0_replica.wait_for_vttablet_state('SERVING')
# run checks now before we start the tablets
utils.validate_topology()
# start a vtgate server too
global vtgate_server, vtgate_port
vtgate_server, vtgate_port = utils.vtgate_start(
cache_ttl='0s', extra_args=vtctld.process_args())
def setUpModule():
global vtgate_server
global vtgate_port
global vtgate_socket_file
global master_start_position
try:
environment.topo_server().setup()
# start mysql instance external to the test
setup_procs = [master_tablet.init_mysql(), replica_tablet.init_mysql()]
utils.wait_procs(setup_procs)
# Start up a master mysql and vttablet
logging.debug('Setting up tablets')
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
master_tablet.init_tablet('master', 'test_keyspace', '0')
replica_tablet.init_tablet('replica', 'test_keyspace', '0')
utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
utils.validate_topology()
master_tablet.create_db('vt_test_keyspace')
master_tablet.create_db('other_database')
replica_tablet.create_db('vt_test_keyspace')
replica_tablet.create_db('other_database')
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])
vtgate_socket_file = environment.tmproot + '/vtgate.sock'
vtgate_server, vtgate_port = utils.vtgate_start(
socket_file=vtgate_socket_file)
master_tablet.start_vttablet()
replica_tablet.start_vttablet()
utils.run_vtctl(['SetReadWrite', master_tablet.tablet_alias])
utils.check_db_read_write(master_tablet.tablet_uid)
for t in [master_tablet, replica_tablet]:
t.reset_replication()
utils.run_vtctl([
'ReparentShard', '-force', 'test_keyspace/0',
master_tablet.tablet_alias
],
auto_log=True)
# reset counter so tests don't assert
tablet.Tablet.tablets_running = 0
master_start_position = _get_master_current_position()
master_tablet.mquery('vt_test_keyspace', _create_vt_insert_test)
master_tablet.mquery('vt_test_keyspace', _create_vt_a)
master_tablet.mquery('vt_test_keyspace', _create_vt_b)
utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias])
utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias])
except:
tearDownModule()
raise
def setUpClass(klass):
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
utils.run_vtctl(['CreateKeyspace',
'--served-from', 'master:test_keyspace,replica:test_keyspace,rdonly:test_keyspace',
'redirected_keyspace'])
shard_0_master.init_tablet( 'master', 'test_keyspace', '-80')
shard_0_replica.init_tablet('spare', 'test_keyspace', '-80')
shard_0_spare.init_tablet( 'spare', 'test_keyspace', '-80')
shard_1_master.init_tablet( 'master', 'test_keyspace', '80-')
shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')
idle.init_tablet('idle')
scrap.init_tablet('idle')
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', 'redirected_keyspace'],
auto_log=True)
# start running all the tablets
for t in [shard_0_master, shard_1_master, shard_1_replica]:
t.create_db('vt_test_keyspace')
t.start_vttablet(wait_for_state=None,
extra_args=utils.vtctld.process_args())
shard_0_replica.create_db('vt_test_keyspace')
shard_0_replica.start_vttablet(extra_args=utils.vtctld.process_args(),
target_tablet_type='replica',
wait_for_state=None)
for t in scrap, idle, shard_0_spare:
t.start_vttablet(wait_for_state=None,
extra_args=utils.vtctld.process_args())
# wait for the right states
for t in [shard_0_master, shard_1_master, shard_1_replica]:
t.wait_for_vttablet_state('SERVING')
for t in [scrap, idle, shard_0_replica, shard_0_spare]:
t.wait_for_vttablet_state('NOT_SERVING')
scrap.scrap()
for t in [shard_0_master, shard_0_replica, shard_0_spare,
shard_1_master, shard_1_replica, idle, scrap]:
t.reset_replication()
utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/-80',
shard_0_master.tablet_alias], auto_log=True)
utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/80-',
shard_1_master.tablet_alias], auto_log=True)
shard_0_replica.wait_for_vttablet_state('SERVING')
# run checks now before we start the tablets
utils.validate_topology()
# start a vtgate server too
global vtgate_server, vtgate_port
vtgate_server, vtgate_port = utils.vtgate_start(
cache_ttl='0s', extra_args=utils.vtctld.process_args())
def setup_tablets():
global vtgate_server
global vtgate_port
# Start up a master mysql and vttablet
logging.debug("Setting up tablets")
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
utils.run_vtctl([
'SetKeyspaceShardingInfo', '-force', 'test_keyspace', 'keyspace_id',
'uint64'
])
shard_0_master.init_tablet('master', keyspace='test_keyspace', shard='-80')
shard_0_replica.init_tablet('replica',
keyspace='test_keyspace',
shard='-80')
shard_1_master.init_tablet('master', keyspace='test_keyspace', shard='80-')
shard_1_replica.init_tablet('replica',
keyspace='test_keyspace',
shard='80-')
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
for t in [
shard_0_master, shard_0_replica, shard_1_master, shard_1_replica
]:
t.create_db('vt_test_keyspace')
t.mquery(shard_0_master.dbname, create_vt_insert_test)
t.mquery(shard_0_master.dbname, create_vt_a)
t.start_vttablet(wait_for_state=None)
for t in [
shard_0_master, shard_0_replica, shard_1_master, shard_1_replica
]:
t.wait_for_vttablet_state('SERVING')
utils.run_vtctl([
'ReparentShard', '-force', 'test_keyspace/-80',
shard_0_master.tablet_alias
],
auto_log=True)
utils.run_vtctl([
'ReparentShard', '-force', 'test_keyspace/80-',
shard_1_master.tablet_alias
],
auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
utils.check_srv_keyspace(
'test_nj', 'test_keyspace', 'Partitions(master): -80 80-\n' +
'Partitions(replica): -80 80-\n' + 'TabletTypes: master,replica')
vtgate_server, vtgate_port = utils.vtgate_start()
vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port,
"test_nj", 30.0)
topology.read_topology(vtgate_client)
def test_vtgate(self):
# Start up a master mysql and vttablet
utils.run_vtctl('CreateKeyspace -force test_keyspace')
utils.run_vtctl('CreateShard -force test_keyspace/0')
tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False)
utils.run_vtctl('RebuildShardGraph test_keyspace/0')
utils.run_vtctl('RebuildKeyspaceGraph test_keyspace')
utils.validate_topology()
# if these statements don't run before the tablet it will wedge waiting for the
# db to become accessible. this is more a bug than a feature.
tablet_62344.mquery("", ["set global read_only = off"])
tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test,
self._populate_vt_select_test)
tablet_62344.start_vttablet()
gate_proc = utils.vtgate_start()
try:
conn = vtgate.connect("localhost:%s"%(utils.vtgate_port_base), "master", "test_keyspace", "0", 2.0)
# _execute
(result, count, lastrow, fields) = conn._execute("select * from vt_select_test", {})
self.assertEqual(count, 4, "want 4, got %d" % (count))
self.assertEqual(len(fields), 2, "want 2, got %d" % (len(fields)))
# _stream_execute
(result, count, lastrow, fields) = conn._stream_execute("select * from vt_select_test", {})
self.assertEqual(len(fields), 2, "want 2, got %d" % (len(fields)))
count = 0
while 1:
r = conn._stream_next()
if not r:
break
count += 1
self.assertEqual(count, 4, "want 4, got %d" % (count))
# begin-rollback
conn.begin()
conn._execute("insert into vt_select_test values(:id, :msg)", {"id": 5, "msg": "test4"})
conn.rollback()
(result, count, lastrow, fields) = conn._execute("select * from vt_select_test", {})
self.assertEqual(count, 4, "want 4, got %d" % (count))
# begin-commit
conn.begin()
conn._execute("insert into vt_select_test values(:id, :msg)", {"id": 5, "msg": "test4"})
conn.commit()
(result, count, lastrow, fields) = conn._execute("select * from vt_select_test", {})
self.assertEqual(count, 5, "want 5, got %d" % (count))
# close
conn.close()
finally:
utils.vtgate_kill(gate_proc)
tablet_62344.kill_vttablet()
def setUpModule():
global vtgate_server
global vtgate_port
global vtgate_socket_file
global master_start_position
try:
environment.topo_server_setup()
# start mysql instance external to the test
setup_procs = [master_tablet.init_mysql(),
replica_tablet.init_mysql()
]
utils.wait_procs(setup_procs)
# Start up a master mysql and vttablet
logging.debug("Setting up tablets")
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
master_tablet.init_tablet('master', 'test_keyspace', '0')
replica_tablet.init_tablet('replica', 'test_keyspace', '0')
utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
utils.validate_topology()
master_tablet.create_db('vt_test_keyspace')
master_tablet.create_db('other_database')
replica_tablet.create_db('vt_test_keyspace')
replica_tablet.create_db('other_database')
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])
vtgate_socket_file = environment.tmproot + '/vtgate.sock'
vtgate_server, vtgate_port = utils.vtgate_start(socket_file=vtgate_socket_file)
master_tablet.start_vttablet()
replica_tablet.start_vttablet()
utils.run_vtctl(['SetReadWrite', master_tablet.tablet_alias])
utils.check_db_read_write(master_tablet.tablet_uid)
for t in [master_tablet, replica_tablet]:
t.reset_replication()
utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/0',
master_tablet.tablet_alias], auto_log=True)
# reset counter so tests don't assert
tablet.Tablet.tablets_running = 0
master_start_position = _get_master_current_position()
master_tablet.mquery('vt_test_keyspace', _create_vt_insert_test)
master_tablet.mquery('vt_test_keyspace', _create_vt_a)
master_tablet.mquery('vt_test_keyspace', _create_vt_b)
utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias])
utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias])
except:
tearDownModule()
raise
def setUp(self):
self.vtgate_server, self.vtgate_port = utils.vtgate_start(cache_ttl='0s')
self.vtgate_client = zkocc.ZkOccConnection("localhost:%u"%self.vtgate_port,
"test_nj", 30.0)
self.vtgate_addrs = None
if vtgate_protocol == VTGATE_PROTOCOL_V1BSON:
global vtgate_addrs
self.vtgate_addrs = {"_vt": ["localhost:%s"%(self.vtgate_port),]}
self.insert_index = 0
def setup_tablets():
global vtgate_server
global vtgate_port
# Start up a master mysql and vttablet
logging.debug("Setting up tablets")
utils.run_vtctl(['CreateKeyspace', KEYSPACE_NAME])
utils.run_vtctl([
'SetKeyspaceShardingInfo', '-force', KEYSPACE_NAME, 'keyspace_id',
'uint64'
])
shard_0_master.init_tablet('master', keyspace=KEYSPACE_NAME, shard='-80')
shard_0_replica.init_tablet('replica', keyspace=KEYSPACE_NAME, shard='-80')
shard_1_master.init_tablet('master', keyspace=KEYSPACE_NAME, shard='80-')
shard_1_replica.init_tablet('replica', keyspace=KEYSPACE_NAME, shard='80-')
utils.run_vtctl(['RebuildKeyspaceGraph', KEYSPACE_NAME], auto_log=True)
for t in [
shard_0_master, shard_0_replica, shard_1_master, shard_1_replica
]:
t.create_db('vt_test_keyspace')
for create_table in create_tables:
t.mquery(shard_0_master.dbname, create_table)
t.start_vttablet(wait_for_state=None)
for t in [
shard_0_master, shard_0_replica, shard_1_master, shard_1_replica
]:
t.wait_for_vttablet_state('SERVING')
utils.run_vtctl([
'ReparentShard', '-force', KEYSPACE_NAME + '/-80',
shard_0_master.tablet_alias
],
auto_log=True)
utils.run_vtctl([
'ReparentShard', '-force', KEYSPACE_NAME + '/80-',
shard_1_master.tablet_alias
],
auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', KEYSPACE_NAME], auto_log=True)
utils.check_srv_keyspace(
'test_nj', KEYSPACE_NAME, 'Partitions(master): -80 80-\n' +
'Partitions(replica): -80 80-\n' + 'TabletTypes: master,replica')
vtgate_server, vtgate_port = utils.vtgate_start()
# FIXME(shrutip): this should be removed once vtgate_cursor's
# dependency on topology goes away.
vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port,
"test_nj", 30.0)
topology.read_topology(vtgate_client)
def setUp(self):
self.vtgate_server, self.vtgate_port = utils.vtgate_start(cache_ttl='0s')
self.vtgate_client = zkocc.ZkOccConnection("localhost:%u"%self.vtgate_port,
"test_nj", 30.0)
self.vtgate_addrs = None
if client_type == VTGATE:
self.vtgate_addrs = {"vt": ["localhost:%s"%(self.vtgate_port),]}
self.insert_index = 0
# Lowering the keyspace refresh throttle so things are testable.
self.throttle_sleep_interval = 0.1
topology.set_keyspace_fetch_throttle(0.01)
def setUp(self):
self.vtgate_server, self.vtgate_port = utils.vtgate_start(cache_ttl='0s')
self.vtgate_client = zkocc.ZkOccConnection("localhost:%u"%self.vtgate_port,
"test_nj", 30.0)
self.vtgate_addrs = None
if vtgate_protocol == VTGATE_PROTOCOL_V1BSON:
global vtgate_addrs
self.vtgate_addrs = {"_vt": ["localhost:%s"%(self.vtgate_port),]}
self.insert_index = 0
# Lowering the keyspace refresh throttle so things are testable.
self.throttle_sleep_interval = 0.1
topology.set_keyspace_fetch_throttle(0.01)
def setUp(self):
global vtgate_server, vtgate_port
self.shard_index = 0
self.replica_tablet = shard_0_replica
self.replica_tablet.kill_vttablet()
self.replica_tablet.start_vttablet(auth=True)
utils.vtgate_kill(vtgate_server)
vtgate_server, vtgate_port = utils.vtgate_start(auth=True)
credentials_file_name = os.path.join(environment.vttop, "test", "test_data", "authcredentials_test.json")
credentials_file = open(credentials_file_name, "r")
credentials = json.load(credentials_file)
self.user = str(credentials.keys()[0])
self.password = str(credentials[self.user][0])
self.secondary_password = str(credentials[self.user][1])
def setup_tablets():
global vtgate_server
global vtgate_port
# Start up a master mysql and vttablet
logging.debug("Setting up tablets")
utils.run_vtctl(['CreateKeyspace', KEYSPACE_NAME])
utils.run_vtctl([
'SetKeyspaceShardingInfo', '-force', KEYSPACE_NAME, 'keyspace_id',
'uint64'
])
shard_0_master.init_tablet('master', keyspace=KEYSPACE_NAME, shard='-80')
shard_0_replica.init_tablet('replica', keyspace=KEYSPACE_NAME, shard='-80')
shard_1_master.init_tablet('master', keyspace=KEYSPACE_NAME, shard='80-')
shard_1_replica.init_tablet('replica', keyspace=KEYSPACE_NAME, shard='80-')
utils.run_vtctl(['RebuildKeyspaceGraph', KEYSPACE_NAME], auto_log=True)
for t in [
shard_0_master, shard_0_replica, shard_1_master, shard_1_replica
]:
t.create_db('vt_test_keyspace')
for create_table in create_tables:
t.mquery(shard_0_master.dbname, create_table)
t.start_vttablet(wait_for_state=None)
for t in [
shard_0_master, shard_0_replica, shard_1_master, shard_1_replica
]:
t.wait_for_vttablet_state('SERVING')
utils.run_vtctl([
'ReparentShard', '-force', KEYSPACE_NAME + '/-80',
shard_0_master.tablet_alias
],
auto_log=True)
utils.run_vtctl([
'ReparentShard', '-force', KEYSPACE_NAME + '/80-',
shard_1_master.tablet_alias
],
auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', KEYSPACE_NAME], auto_log=True)
utils.check_srv_keyspace(
'test_nj', KEYSPACE_NAME, 'Partitions(master): -80 80-\n' +
'Partitions(replica): -80 80-\n' + 'TabletTypes: master,replica')
vtgate_server, vtgate_port = utils.vtgate_start()
def set_up(self):
try:
environment.topo_server().setup()
utils.wait_procs([t.init_mysql() for t in self.tablets])
utils.run_vtctl(['CreateKeyspace', self.keyspace])
utils.run_vtctl([
'SetKeyspaceShardingInfo', '-force', self.keyspace,
'keyspace_id', 'uint64'
])
for t in self.tablets:
t.init_tablet(t.type, keyspace=self.keyspace, shard=t.shard)
utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace],
auto_log=True)
for t in self.tablets:
t.create_db('vt_' + self.keyspace)
t.start_vttablet(
wait_for_state=None,
extra_args=['-queryserver-config-schema-reload-time', '1'],
)
for t in self.tablets:
t.wait_for_vttablet_state('SERVING')
for t in self.tablets:
if t.type == "master":
utils.run_vtctl([
'ReparentShard', '-force',
self.keyspace + '/' + t.shard, t.tablet_alias
],
auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace],
auto_log=True)
if self.schema:
utils.run_vtctl([
'ApplySchemaKeyspace', '-simple', '-sql', self.schema,
self.keyspace
])
if self.vschema:
if self.vschema[0] == '{':
utils.run_vtctl(['ApplyVSchema', "-vschema", self.vschema])
else:
utils.run_vtctl(
['ApplyVSchema', "-vschema_file", self.vschema])
self.vtgate_server, self.vtgate_port = utils.vtgate_start(
cache_ttl='500s')
vtgate_client = zkocc.ZkOccConnection(
"localhost:%u" % self.vtgate_port, "test_nj", 30.0)
topology.read_topology(vtgate_client)
except:
self.shutdown()
raise
def setUp(self):
global vtgate_server, vtgate_port
self.shard_index = 0
self.replica_tablet = shard_0_replica
self.replica_tablet.kill_vttablet()
self.replica_tablet.start_vttablet(auth=True)
utils.vtgate_kill(vtgate_server)
vtgate_server, vtgate_port = utils.vtgate_start(auth=True)
credentials_file_name = os.path.join(environment.vttop, 'test', 'test_data',
'authcredentials_test.json')
credentials_file = open(credentials_file_name, 'r')
credentials = json.load(credentials_file)
self.user = str(credentials.keys()[0])
self.password = str(credentials[self.user][0])
self.secondary_password = str(credentials[self.user][1])
def setUpModule():
global vtgate_server, vtgate_port
try:
environment.topo_server().setup()
setup_topology()
# start mysql instance external to the test
global __tablets
setup_procs = []
for tablet in __tablets:
setup_procs.append(tablet.init_mysql())
utils.wait_procs(setup_procs)
create_db()
start_tablets()
vtgate_server, vtgate_port = utils.vtgate_start()
except:
tearDownModule()
raise
def setUpModule():
global vtgate_server, vtgate_port
try:
environment.topo_server().setup()
setup_topology()
# start mysql instance external to the test
global __tablets
setup_procs = []
for tablet in __tablets:
setup_procs.append(tablet.init_mysql())
utils.wait_procs(setup_procs)
create_db()
start_tablets()
vtgate_server, vtgate_port = utils.vtgate_start()
except:
tearDownModule()
raise
def setup_tablets():
global vtgate_server
global vtgate_port
# Start up a master mysql and vttablet
logging.debug("Setting up tablets")
utils.run_vtctl(['CreateKeyspace', KEYSPACE_NAME])
utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', KEYSPACE_NAME,
'keyspace_id', 'uint64'])
shard_0_master.init_tablet('master', keyspace=KEYSPACE_NAME, shard='-80')
shard_0_replica.init_tablet('replica', keyspace=KEYSPACE_NAME, shard='-80')
shard_1_master.init_tablet('master', keyspace=KEYSPACE_NAME, shard='80-')
shard_1_replica.init_tablet('replica', keyspace=KEYSPACE_NAME, shard='80-')
utils.run_vtctl(['RebuildKeyspaceGraph', KEYSPACE_NAME], auto_log=True)
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.create_db('vt_test_keyspace')
for create_table in create_tables:
t.mquery(shard_0_master.dbname, create_table)
t.start_vttablet(wait_for_state=None)
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.wait_for_vttablet_state('SERVING')
utils.run_vtctl(['ReparentShard', '-force', KEYSPACE_NAME+'/-80',
shard_0_master.tablet_alias], auto_log=True)
utils.run_vtctl(['ReparentShard', '-force', KEYSPACE_NAME+'/80-',
shard_1_master.tablet_alias], auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', KEYSPACE_NAME],
auto_log=True)
utils.check_srv_keyspace('test_nj', KEYSPACE_NAME,
'Partitions(master): -80 80-\n' +
'Partitions(replica): -80 80-\n' +
'TabletTypes: master,replica')
vtgate_server, vtgate_port = utils.vtgate_start()
# FIXME(shrutip): this should be removed once vtgate_cursor's
# dependency on topology goes away.
vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port,
"test_nj", 30.0)
topology.read_topology(vtgate_client)
def setUpClass(klass):
utils.run_vtctl('CreateKeyspace test_keyspace')
shard_0_master.init_tablet( 'master', 'test_keyspace', '-80')
shard_0_replica.init_tablet('spare', 'test_keyspace', '-80')
shard_0_spare.init_tablet('spare', 'test_keyspace', '-80')
shard_1_master.init_tablet( 'master', 'test_keyspace', '80-')
shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')
idle.init_tablet('idle')
scrap.init_tablet('idle')
utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)
for t in [shard_0_master, shard_1_master, shard_1_replica]:
t.create_db('vt_test_keyspace')
t.start_vttablet(extra_args=vtctld.process_args())
shard_0_replica.create_db('vt_test_keyspace')
shard_0_replica.start_vttablet(extra_args=vtctld.process_args(),
target_tablet_type='replica',
wait_for_state='NOT_SERVING')
for t in scrap, idle, shard_0_spare:
t.start_vttablet(wait_for_state='NOT_SERVING',
extra_args=vtctld.process_args())
scrap.scrap()
for t in [shard_0_master, shard_0_replica, shard_0_spare,
shard_1_master, shard_1_replica, idle, scrap]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/-80 ' +
shard_0_master.tablet_alias, auto_log=True)
utils.run_vtctl('ReparentShard -force test_keyspace/80- ' +
shard_1_master.tablet_alias, auto_log=True)
shard_0_replica.wait_for_vttablet_state('SERVING')
# run checks now before we start the tablets
utils.validate_topology()
# start a vtgate server too
global vtgate_server, vtgate_port
vtgate_server, vtgate_port = utils.vtgate_start(
cache_ttl='0s', extra_args=vtctld.process_args())
def setUpModule():
global vtgate_server
global vtgate_port
try:
environment.topo_server().setup()
setup_procs = [
shard_0_master.init_mysql(),
shard_0_rdonly.init_mysql(),
shard_1_master.init_mysql(),
shard_1_rdonly.init_mysql(),
]
utils.Vtctld().start()
vtgate_server, vtgate_port = utils.vtgate_start()
utils.wait_procs(setup_procs)
except:
tearDownModule()
raise
def setup_tablets():
global vtgate_server
global vtgate_port
# Start up a master mysql and vttablet
logging.debug("Setting up tablets")
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', 'test_keyspace',
'keyspace_id', 'uint64'])
shard_0_master.init_tablet('master', keyspace='test_keyspace', shard='-80')
shard_0_replica.init_tablet('replica', keyspace='test_keyspace', shard='-80')
shard_1_master.init_tablet('master', keyspace='test_keyspace', shard='80-')
shard_1_replica.init_tablet('replica', keyspace='test_keyspace', shard='80-')
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.create_db('vt_test_keyspace')
t.mquery(shard_0_master.dbname, create_vt_insert_test)
t.mquery(shard_0_master.dbname, create_vt_a)
t.start_vttablet(wait_for_state=None)
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.wait_for_vttablet_state('SERVING')
utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/-80',
shard_0_master.tablet_alias], auto_log=True)
utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/80-',
shard_1_master.tablet_alias], auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'],
auto_log=True)
utils.check_srv_keyspace('test_nj', 'test_keyspace',
'Partitions(master): -80 80-\n' +
'Partitions(replica): -80 80-\n' +
'TabletTypes: master,replica')
vtgate_server, vtgate_port = utils.vtgate_start()
vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port,
"test_nj", 30.0)
topology.read_topology(vtgate_client)
def setUpModule():
global vtgate_server
global vtgate_port
try:
environment.topo_server_setup()
# start mysql instance external to the test
setup_procs = [master_tablet.init_mysql(), replica_tablet.init_mysql()]
utils.wait_procs(setup_procs)
# Start up a master mysql and vttablet
logging.debug("Setting up tablets")
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
master_tablet.init_tablet('master', 'test_keyspace', '0')
replica_tablet.init_tablet('replica', 'test_keyspace', '0')
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])
utils.validate_topology()
master_tablet.populate('vt_test_keyspace', create_vt_insert_test)
replica_tablet.populate('vt_test_keyspace', create_vt_insert_test)
vtgate_server, vtgate_port = utils.vtgate_start()
master_tablet.start_vttablet(memcache=True, wait_for_state=None)
replica_tablet.start_vttablet(memcache=True, wait_for_state=None)
master_tablet.wait_for_vttablet_state('SERVING')
replica_tablet.wait_for_vttablet_state('SERVING')
utils.run_vtctl([
'ReparentShard', '-force', 'test_keyspace/0',
master_tablet.tablet_alias
],
auto_log=True)
utils.validate_topology()
# restart the replica tablet so the stats are reset
replica_tablet.kill_vttablet()
replica_tablet.start_vttablet(memcache=True)
except:
tearDownModule()
raise
def setUpModule():
global keyspace_env
global shard_0_master
global shard_1_master
global lookup_master
global vtgate_server
global vtgate_port
logging.debug("in setUpModule")
try:
environment.topo_server().setup()
logging.debug("Setting up tablets")
keyspace_env = keyspace_util.TestEnv()
keyspace_env.launch(
"user",
shards=["-80", "80-"],
ddls=[
create_vt_user,
create_vt_user2,
create_vt_user_extra,
create_vt_music,
create_vt_music_extra,
],
)
keyspace_env.launch(
"lookup",
ddls=[
create_vt_user_idx,
create_music_user_map,
create_name_user2_map,
],
)
shard_0_master = keyspace_env.tablet_map["user.-80.master"]
shard_1_master = keyspace_env.tablet_map["user.80-.master"]
lookup_master = keyspace_env.tablet_map["lookup.0.master"]
utils.apply_vschema(schema)
vtgate_server, vtgate_port = utils.vtgate_start()
except:
tearDownModule()
raise
def setUpModule():
global keyspace_env
global shard_0_master
global shard_1_master
global lookup_master
global vtgate_server
global vtgate_port
logging.debug("in setUpModule")
try:
environment.topo_server().setup()
logging.debug("Setting up tablets")
keyspace_env = keyspace_util.TestEnv()
keyspace_env.launch(
"user",
shards=["-80", "80-"],
ddls=[
create_vt_user,
create_vt_user2,
create_vt_user_extra,
create_vt_music,
create_vt_music_extra,
],
)
keyspace_env.launch(
"lookup",
ddls=[
create_vt_user_idx,
create_music_user_map,
create_name_user2_map,
],
)
shard_0_master = keyspace_env.tablet_map["user.-80.master"]
shard_1_master = keyspace_env.tablet_map["user.80-.master"]
lookup_master = keyspace_env.tablet_map["lookup.0.master"]
utils.apply_vschema(schema)
vtgate_server, vtgate_port = utils.vtgate_start()
except:
tearDownModule()
raise
def main():
parser = optparse.OptionParser(usage="usage: %prog [options]")
utils.add_options(parser)
(options, args) = parser.parse_args()
options.debug = True
utils.set_options(options)
env = keyspace_util.TestEnv()
vtgate_server = None
try:
environment.topo_server().setup()
env.launch(
"user",
shards=["-80", "80-"],
ddls=[
'create table user(user_id bigint, name varchar(128), primary key(user_id))',
'create table user_extra(user_id bigint, extra varchar(128), primary key(user_id))',
'create table music(user_id bigint, music_id bigint, primary key(user_id, music_id))',
'create table music_extra(music_id bigint, keyspace_id bigint unsigned, primary key(music_id))',
],
)
env.launch(
"lookup",
ddls=[
'create table user_idx(user_id bigint not null auto_increment, primary key(user_id))',
'create table name_user_idx(name varchar(128), user_id bigint, primary key(name, user_id))',
'create table music_user_idx(music_id bigint not null auto_increment, user_id bigint, primary key(music_id))',
],
)
utils.apply_vschema(vschema)
vtgate_server, vtgate_port = utils.vtgate_start(cache_ttl='500s')
utils.Vtctld().start()
print "vtgate:", vtgate_port
print "vtctld:", utils.vtctld.port
utils.pause("the cluster is up, press enter to shut it down...")
finally:
utils.vtgate_kill(vtgate_server)
env.teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
environment.topo_server().teardown()
def setUpModule():
global vtgate_server
global vtgate_port
try:
environment.topo_server_setup()
# start mysql instance external to the test
setup_procs = [master_tablet.init_mysql(),
replica_tablet.init_mysql()]
utils.wait_procs(setup_procs)
# Start up a master mysql and vttablet
logging.debug("Setting up tablets")
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
master_tablet.init_tablet('master', 'test_keyspace', '0')
replica_tablet.init_tablet('replica', 'test_keyspace', '0')
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])
utils.validate_topology()
master_tablet.populate('vt_test_keyspace', create_vt_insert_test)
replica_tablet.populate('vt_test_keyspace', create_vt_insert_test)
vtgate_server, vtgate_port = utils.vtgate_start()
master_tablet.start_vttablet(memcache=True, wait_for_state=None)
replica_tablet.start_vttablet(memcache=True, wait_for_state=None)
master_tablet.wait_for_vttablet_state('SERVING')
replica_tablet.wait_for_vttablet_state('SERVING')
utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/0',
master_tablet.tablet_alias], auto_log=True)
utils.validate_topology()
# restart the replica tablet so the stats are reset
replica_tablet.kill_vttablet()
replica_tablet.start_vttablet(memcache=True)
except:
tearDownModule()
raise
def main():
parser = optparse.OptionParser(usage="usage: %prog [options]")
utils.add_options(parser)
(options, args) = parser.parse_args()
options.debug = True
utils.set_options(options)
env = keyspace_util.TestEnv()
vtgate_server=None
try:
environment.topo_server().setup()
env.launch(
"user",
shards=["-80", "80-"],
ddls=[
'create table user(user_id bigint, name varchar(128), primary key(user_id))',
'create table user_extra(user_id bigint, extra varchar(128), primary key(user_id))',
'create table music(user_id bigint, music_id bigint, primary key(user_id, music_id))',
'create table music_extra(music_id bigint, keyspace_id bigint unsigned, primary key(music_id))',
],
)
env.launch(
"lookup",
ddls=[
'create table user_idx(user_id bigint not null auto_increment, primary key(user_id))',
'create table name_user_idx(name varchar(128), user_id bigint, primary key(name, user_id))',
'create table music_user_idx(music_id bigint not null auto_increment, user_id bigint, primary key(music_id))',
],
)
utils.apply_vschema(vschema)
vtgate_server, vtgate_port = utils.vtgate_start(cache_ttl='500s')
utils.Vtctld().start()
print "vtgate:", vtgate_port
print "vtctld:", utils.vtctld.port
utils.pause("the cluster is up, press enter to shut it down...")
finally:
utils.vtgate_kill(vtgate_server)
env.teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
environment.topo_server().teardown()
def setup_tablets():
global vtgate_server
global vtgate_port
# Start up a master mysql and vttablet
logging.debug("Setting up tablets")
utils.run_vtctl(['CreateKeyspace', KEYSPACE_NAME])
utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', KEYSPACE_NAME,
'keyspace_id', 'uint64'])
shard_0_master.init_tablet('master', keyspace=KEYSPACE_NAME, shard='-80')
shard_0_replica.init_tablet('replica', keyspace=KEYSPACE_NAME, shard='-80')
shard_1_master.init_tablet('master', keyspace=KEYSPACE_NAME, shard='80-')
shard_1_replica.init_tablet('replica', keyspace=KEYSPACE_NAME, shard='80-')
utils.run_vtctl(['RebuildKeyspaceGraph', KEYSPACE_NAME], auto_log=True)
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.create_db('vt_test_keyspace')
for create_table in create_tables:
t.mquery(shard_0_master.dbname, create_table)
t.start_vttablet(wait_for_state=None)
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.wait_for_vttablet_state('SERVING')
utils.run_vtctl(['ReparentShard', '-force', KEYSPACE_NAME+'/-80',
shard_0_master.tablet_alias], auto_log=True)
utils.run_vtctl(['ReparentShard', '-force', KEYSPACE_NAME+'/80-',
shard_1_master.tablet_alias], auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', KEYSPACE_NAME],
auto_log=True)
utils.check_srv_keyspace('test_nj', KEYSPACE_NAME,
'Partitions(master): -80 80-\n' +
'Partitions(replica): -80 80-\n' +
'TabletTypes: master,replica')
vtgate_server, vtgate_port = utils.vtgate_start()
def setup_tablets():
global vtgate_server
global vtgate_port
# Start up a master mysql and vttablet
logging.debug("Setting up tablets")
utils.run_vtctl(["CreateKeyspace", "test_keyspace"])
utils.run_vtctl(["SetKeyspaceShardingInfo", "-force", "test_keyspace", "keyspace_id", "uint64"])
shard_0_master.init_tablet("master", keyspace="test_keyspace", shard="-80")
shard_0_replica.init_tablet("replica", keyspace="test_keyspace", shard="-80")
shard_1_master.init_tablet("master", keyspace="test_keyspace", shard="80-")
shard_1_replica.init_tablet("replica", keyspace="test_keyspace", shard="80-")
utils.run_vtctl(["RebuildKeyspaceGraph", "test_keyspace"], auto_log=True)
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.create_db("vt_test_keyspace")
t.mquery(shard_0_master.dbname, create_vt_insert_test)
t.mquery(shard_0_master.dbname, create_vt_a)
t.start_vttablet(wait_for_state=None)
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.wait_for_vttablet_state("SERVING")
utils.run_vtctl(["ReparentShard", "-force", "test_keyspace/-80", shard_0_master.tablet_alias], auto_log=True)
utils.run_vtctl(["ReparentShard", "-force", "test_keyspace/80-", shard_1_master.tablet_alias], auto_log=True)
utils.run_vtctl(["RebuildKeyspaceGraph", "test_keyspace"], auto_log=True)
utils.check_srv_keyspace(
"test_nj",
"test_keyspace",
"Partitions(master): -80 80-\n" + "Partitions(replica): -80 80-\n" + "TabletTypes: master,replica",
)
vtgate_server, vtgate_port = utils.vtgate_start()
vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port, "test_nj", 30.0)
topology.read_topology(vtgate_client)
def setup_tablets():
global vtgate_server
global vtgate_port
# Start up a master mysql and vttablet
logging.debug("Setting up tablets")
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
shard_0_master.init_tablet('master', keyspace='test_keyspace', shard='0')
shard_0_replica.init_tablet('replica', keyspace='test_keyspace', shard='0')
shard_1_master.init_tablet('master', keyspace='test_keyspace', shard='1')
shard_1_replica.init_tablet('replica', keyspace='test_keyspace', shard='1')
utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'], auto_log=True)
utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/1'], auto_log=True)
utils.validate_topology()
shard_0_master.create_db(shard_0_master.dbname)
shard_0_replica.create_db(shard_0_master.dbname)
shard_1_master.create_db(shard_0_master.dbname)
shard_1_replica.create_db(shard_0_master.dbname)
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.mquery(shard_0_master.dbname, create_vt_insert_test)
t.mquery(shard_0_master.dbname, create_vt_a)
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
vtgate_server, vtgate_port = utils.vtgate_start()
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.start_vttablet(wait_for_state=None)
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.wait_for_vttablet_state('SERVING')
utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/0',
shard_0_master.tablet_alias], auto_log=True)
utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/1',
shard_1_master.tablet_alias], auto_log=True)
def setup_tablets():
global vtgate_server
global vtgate_port
# Start up a master mysql and vttablet
logging.debug("Setting up tablets")
utils.run_vtctl(["CreateKeyspace", KEYSPACE_NAME])
utils.run_vtctl(["SetKeyspaceShardingInfo", "-force", KEYSPACE_NAME, "keyspace_id", "uint64"])
shard_0_master.init_tablet("master", keyspace=KEYSPACE_NAME, shard="-80")
shard_0_replica.init_tablet("replica", keyspace=KEYSPACE_NAME, shard="-80")
shard_1_master.init_tablet("master", keyspace=KEYSPACE_NAME, shard="80-")
shard_1_replica.init_tablet("replica", keyspace=KEYSPACE_NAME, shard="80-")
utils.run_vtctl(["RebuildKeyspaceGraph", KEYSPACE_NAME], auto_log=True)
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.create_db("vt_test_keyspace")
for create_table in create_tables:
t.mquery(shard_0_master.dbname, create_table)
t.start_vttablet(wait_for_state=None)
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.wait_for_vttablet_state("SERVING")
utils.run_vtctl(["ReparentShard", "-force", KEYSPACE_NAME + "/-80", shard_0_master.tablet_alias], auto_log=True)
utils.run_vtctl(["ReparentShard", "-force", KEYSPACE_NAME + "/80-", shard_1_master.tablet_alias], auto_log=True)
utils.run_vtctl(["RebuildKeyspaceGraph", KEYSPACE_NAME], auto_log=True)
utils.check_srv_keyspace(
"test_nj",
KEYSPACE_NAME,
"Partitions(master): -80 80-\n" + "Partitions(replica): -80 80-\n" + "TabletTypes: master,replica",
)
vtgate_server, vtgate_port = utils.vtgate_start()
def setUpModule():
global vtgate_server, vtgate_port
logging.debug("in setUpModule")
try:
environment.topo_server().setup()
setup_topology()
# start mysql instance external to the test
global __tablets
setup_procs = []
for tablet in __tablets:
setup_procs.append(tablet.init_mysql())
utils.wait_procs(setup_procs)
create_db()
start_tablets()
vtgate_server, vtgate_port = utils.vtgate_start()
# FIXME(shrutip): this should be removed once vtgate_cursor's
# dependency on topology goes away.
vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port,
"test_nj", 30.0)
topology.read_topology(vtgate_client)
except:
tearDownModule()
raise
def restart_vtgate(extra_args={}):
global vtgate_server, vtgate_port
utils.vtgate_kill(vtgate_server)
vtgate_server, vtgate_port = utils.vtgate_start(vtgate_port, extra_args=extra_args)
self.fail("Communication with shard %s replica failed with error %s" %
(shard_names[self.shard_index], str(e)))
def test_vtgate_restart_read(self):
global vtgate_server, vtgate_port
try:
vtgate_conn = get_connection()
except Exception, e:
self.fail("Connection to vtgate failed with error %s" % (str(e)))
utils.vtgate_kill(vtgate_server)
with self.assertRaises(dbexceptions.OperationalError):
vtgate_conn._execute(
"select 1 from vt_insert_test", {},
KEYSPACE_NAME, 'replica',
keyranges=[self.keyrange])
vtgate_server, vtgate_port = utils.vtgate_start(vtgate_port)
vtgate_conn = get_connection()
try:
results = vtgate_conn._execute(
"select 1 from vt_insert_test", {},
KEYSPACE_NAME, 'replica',
keyranges=[self.keyrange])
except Exception, e:
self.fail("Communication with shard %s replica failed with error %s" %
(shard_names[self.shard_index], str(e)))
def test_tablet_restart_stream_execute(self):
try:
vtgate_conn = get_connection()
except Exception, e:
self.fail("Connection to vtgate failed with error %s" % (str(e)))
def test_vtgate(self):
# Start up a master mysql and vttablet
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
utils.run_vtctl(['CreateShard', 'test_keyspace/0'])
tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False)
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])
utils.validate_topology()
srvShard = utils.run_vtctl_json(
['GetSrvShard', 'test_nj', 'test_keyspace/0'])
self.assertEqual(srvShard['MasterCell'], 'test_nj')
# if these statements don't run before the tablet it will wedge
# waiting for the db to become accessible. this is more a bug than
# a feature.
tablet_62344.mquery("", ["set global read_only = off"])
tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test,
self._populate_vt_select_test)
tablet_62344.start_vttablet()
gate_proc, gate_port = utils.vtgate_start()
conn = vtgate.connect("localhost:%s" % (gate_port), "master",
"test_keyspace", "0", 2.0)
# _execute
(result, count, lastrow,
fields) = conn._execute("select * from vt_select_test", {})
self.assertEqual(count, 4, "want 4, got %d" % (count))
self.assertEqual(len(fields), 2, "want 2, got %d" % (len(fields)))
# _execute_batch
queries = [
"select * from vt_select_test where id = :id",
"select * from vt_select_test where id = :id",
]
bindvars = [
{
"id": 1
},
{
"id": 2
},
]
rowsets = conn._execute_batch(queries, bindvars)
self.assertEqual(rowsets[0][0][0][0], 1)
self.assertEqual(rowsets[1][0][0][0], 2)
# _stream_execute
(result, count, lastrow,
fields) = conn._stream_execute("select * from vt_select_test", {})
self.assertEqual(len(fields), 2, "want 2, got %d" % (len(fields)))
count = 0
while 1:
r = conn._stream_next()
if not r:
break
count += 1
self.assertEqual(count, 4, "want 4, got %d" % (count))
# begin-rollback
conn.begin()
conn._execute("insert into vt_select_test values(:id, :msg)", {
"id": 5,
"msg": "test4"
})
conn.rollback()
(result, count, lastrow,
fields) = conn._execute("select * from vt_select_test", {})
self.assertEqual(count, 4, "want 4, got %d" % (count))
# begin-commit
conn.begin()
conn._execute("insert into vt_select_test values(:id, :msg)", {
"id": 5,
"msg": "test4"
})
conn.commit()
(result, count, lastrow,
fields) = conn._execute("select * from vt_select_test", {})
self.assertEqual(count, 5, "want 5, got %d" % (count))
# error on dml. We still need to get a transaction id
conn.begin()
with self.assertRaises(dbexceptions.IntegrityError):
conn._execute("insert into vt_select_test values(:id, :msg)", {
"id": 5,
"msg": "test4"
})
self.assertTrue(conn.session["ShardSessions"][0]["TransactionId"] != 0)
conn.commit()
# interleaving
conn2 = vtgate.connect("localhost:%s" % (gate_port), "master",
"test_keyspace", "0", 2.0)
thd = threading.Thread(target=self._query_lots, args=(conn2, ))
thd.start()
for i in xrange(250):
(result, count, lastrow, fields) = conn._execute(
"select id from vt_select_test where id = 2", {})
self.assertEqual(result, [(2, )])
if i % 10 == 0:
conn._stream_execute(
"select id from vt_select_test where id = 3", {})
while 1:
result = conn._stream_next()
if not result:
break
self.assertEqual(result, (3, ))
thd.join()
# close
conn.close()
utils.vtgate_kill(gate_proc)
tablet_62344.kill_vttablet()
def setUp(self): environment.topo_server().wipe() self.vtgate_zk, self.vtgate_zk_port = utils.vtgate_start()
def test_sharding(self):
shard_0_master.init_tablet('master', 'test_keyspace', '-80')
shard_0_replica.init_tablet('replica', 'test_keyspace', '-80')
shard_1_master.init_tablet('master', 'test_keyspace', '80-')
shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')
utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)
# run checks now before we start the tablets
utils.validate_topology()
# create databases, start the tablets, wait for them to start
for t in [
shard_0_master, shard_0_replica, shard_1_master,
shard_1_replica
]:
t.create_db('vt_test_keyspace')
t.start_vttablet(wait_for_state=None)
for t in [
shard_0_master, shard_0_replica, shard_1_master,
shard_1_replica
]:
t.wait_for_vttablet_state('SERVING')
# apply the schema on the first shard through vtctl, so all tablets
# are the same (replication is not enabled yet, so allow_replication=false
# is just there to be tested)
utils.run_vtctl([
'ApplySchema', '-stop-replication',
'-sql=' + create_vt_select_test.replace("\n", ""),
shard_0_master.tablet_alias
])
utils.run_vtctl([
'ApplySchema', '-stop-replication',
'-sql=' + create_vt_select_test.replace("\n", ""),
shard_0_replica.tablet_alias
])
if environment.topo_server_implementation == 'zookeeper':
# start zkocc, we'll use it later, indirectly with the vtdb-zkocc driver
zkocc_server = utils.zkocc_start()
# start vtgate, we'll use it later
vtgate_server, vtgate_port = utils.vtgate_start()
for t in [
shard_0_master, shard_0_replica, shard_1_master,
shard_1_replica
]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/-80 ' +
shard_0_master.tablet_alias,
auto_log=True)
utils.run_vtctl('ReparentShard -force test_keyspace/80- ' +
shard_1_master.tablet_alias,
auto_log=True)
# apply the schema on the second shard using a simple schema upgrade
utils.run_vtctl([
'ApplySchemaShard', '-simple',
'-sql=' + create_vt_select_test_reverse.replace("\n", ""),
'test_keyspace/80-'
])
# insert some values directly (db is RO after minority reparent)
# FIXME(alainjobart) these values don't match the shard map
utils.run_vtctl('SetReadWrite ' + shard_0_master.tablet_alias)
utils.run_vtctl('SetReadWrite ' + shard_1_master.tablet_alias)
shard_0_master.mquery(
'vt_test_keyspace',
"insert into vt_select_test (id, msg) values (1, 'test 1')",
write=True)
shard_1_master.mquery(
'vt_test_keyspace',
"insert into vt_select_test (id, msg) values (10, 'test 10')",
write=True)
utils.validate_topology(ping_tablets=True)
utils.pause("Before the sql scatter query")
# note the order of the rows is not guaranteed, as the go routines
# doing the work can go out of order
self._check_rows(["Index\tid\tmsg", "1\ttest 1", "10\ttest 10"])
# write a value, re-read them all
utils.vtclient2(
3803,
"/test_nj/test_keyspace/master",
"insert into vt_select_test (id, msg) values (:keyspace_id, 'test 2')",
bindvars='{"keyspace_id": 2}',
driver="vtdb",
verbose=True)
self._check_rows(
["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"])
# make sure the '2' value was written on first shard
rows = shard_0_master.mquery(
'vt_test_keyspace',
"select id, msg from vt_select_test order by id")
self.assertEqual(rows, (
(1, 'test 1'),
(2, 'test 2'),
), 'wrong mysql_query output: %s' % str(rows))
utils.pause("After db writes")
# now use various topo servers and streaming or both for the same query
self._check_rows(
["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"],
driver="vtdb-streaming")
if environment.topo_server_implementation == 'zookeeper':
self._check_rows(
["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"],
driver="vtdb-zk")
self._check_rows(
["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"],
driver="vtdb-zk-streaming")
self._check_rows(
["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"],
driver="vtdb-zkocc")
self._check_rows(
["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"],
driver="vtdb-zkocc-streaming")
# make sure the schema checking works
self._check_rows_schema_diff("vtdb")
if environment.topo_server_implementation == 'zookeeper':
self._check_rows_schema_diff("vtdb-zk")
self._check_rows_schema_diff("vtdb-zkocc")
# throw in some schema validation step
# we created the schema differently, so it should show
utils.run_vtctl('ValidateSchemaShard test_keyspace/-80')
utils.run_vtctl('ValidateSchemaShard test_keyspace/80-')
out, err = utils.run_vtctl('ValidateSchemaKeyspace test_keyspace',
trap_output=True,
raise_on_error=False)
if "test_nj-0000062344 and test_nj-0000062346 disagree on schema for table vt_select_test:\nCREATE TABLE" not in err or \
"test_nj-0000062344 and test_nj-0000062347 disagree on schema for table vt_select_test:\nCREATE TABLE" not in err:
self.fail('wrong ValidateSchemaKeyspace output: ' + err)
# validate versions
utils.run_vtctl('ValidateVersionShard test_keyspace/-80',
auto_log=True)
utils.run_vtctl('ValidateVersionKeyspace test_keyspace', auto_log=True)
# show and validate permissions
utils.run_vtctl('GetPermissions test_nj-0000062344', auto_log=True)
utils.run_vtctl('ValidatePermissionsShard test_keyspace/-80',
auto_log=True)
utils.run_vtctl('ValidatePermissionsKeyspace test_keyspace',
auto_log=True)
if environment.topo_server_implementation == 'zookeeper':
# and create zkns on this complex keyspace, make sure a few files are created
utils.run_vtctl('ExportZknsForKeyspace test_keyspace')
out, err = utils.run(environment.binary_path('zk') +
' ls -R /zk/test_nj/zk?s/vt/test_keysp*',
trap_output=True)
lines = out.splitlines()
for base in ['-80', '80-']:
for db_type in ['master', 'replica']:
for sub_path in ['', '.vdns', '/0', '/_vtocc.vdns']:
expected = '/zk/test_nj/zkns/vt/test_keyspace/' + base + '/' + db_type + sub_path
if expected not in lines:
self.fail('missing zkns part:\n%s\nin:%s' %
(expected, out))
# now try to connect using the python client and shard-aware connection
# to both shards
# first get the topology and check it
vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port,
"test_nj", 30.0)
topology.read_keyspaces(vtgate_client)
shard_0_master_addrs = topology.get_host_port_by_name(
vtgate_client, "test_keyspace.-80.master:_vtocc")
if len(shard_0_master_addrs) != 1:
self.fail(
'topology.get_host_port_by_name failed for "test_keyspace.-80.master:_vtocc", got: %s'
% " ".join([
"%s:%u(%s)" % (h, p, str(e))
for (h, p, e) in shard_0_master_addrs
]))
logging.debug(
"shard 0 master addrs: %s", " ".join([
"%s:%u(%s)" % (h, p, str(e))
for (h, p, e) in shard_0_master_addrs
]))
# connect to shard -80
conn = tablet3.TabletConnection(
"%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
"", "test_keyspace", "-80", 10.0)
conn.dial()
(results, rowcount, lastrowid, fields) = conn._execute(
"select id, msg from vt_select_test order by id", {})
self.assertEqual(results, [
(1, 'test 1'),
(2, 'test 2'),
], 'wrong conn._execute output: %s' % str(results))
# connect to shard 80-
shard_1_master_addrs = topology.get_host_port_by_name(
vtgate_client, "test_keyspace.80-.master:_vtocc")
conn = tablet3.TabletConnection(
"%s:%u" % (shard_1_master_addrs[0][0], shard_1_master_addrs[0][1]),
"", "test_keyspace", "80-", 10.0)
conn.dial()
(results, rowcount, lastrowid, fields) = conn._execute(
"select id, msg from vt_select_test order by id", {})
self.assertEqual(results, [
(10, 'test 10'),
], 'wrong conn._execute output: %s' % str(results))
vtgate_client.close()
# try to connect with bad shard
try:
conn = tablet3.TabletConnection(
"localhost:%u" % shard_0_master.port, "", "test_keyspace",
"-90", 10.0)
conn.dial()
self.fail('expected an exception')
except Exception as e:
if "fatal: Shard mismatch, expecting -80, received -90" not in str(
e):
self.fail('unexpected exception: ' + str(e))
utils.vtgate_kill(vtgate_server)
if environment.topo_server_implementation == 'zookeeper':
utils.kill_sub_process(zkocc_server)
tablet.kill_tablets(
[shard_0_master, shard_0_replica, shard_1_master, shard_1_replica])
def restart_vtgate(extra_args={}):
global vtgate_server, vtgate_port
utils.vtgate_kill(vtgate_server)
vtgate_server, vtgate_port = utils.vtgate_start(vtgate_port,
extra_args=extra_args)
def setUp(self): environment.topo_server().wipe() self.vtgate_zk, self.vtgate_zk_port = utils.vtgate_start()
"Communication with shard %s replica failed with error %s" %
(shard_names[self.shard_index], str(e)))
def test_vtgate_restart_read(self):
global vtgate_server, vtgate_port
try:
vtgate_conn = get_connection()
except Exception, e:
self.fail("Connection to vtgate failed with error %s" % (str(e)))
utils.vtgate_kill(vtgate_server)
with self.assertRaises(dbexceptions.OperationalError):
vtgate_conn._execute("select 1 from vt_insert_test", {},
KEYSPACE_NAME,
'replica',
keyranges=[self.keyrange])
vtgate_server, vtgate_port = utils.vtgate_start(vtgate_port)
vtgate_conn = get_connection()
try:
results = vtgate_conn._execute("select 1 from vt_insert_test", {},
KEYSPACE_NAME,
'replica',
keyranges=[self.keyrange])
except Exception, e:
self.fail(
"Communication with shard %s replica failed with error %s" %
(shard_names[self.shard_index], str(e)))
def test_tablet_restart_stream_execute(self):
try:
vtgate_conn = get_connection()
except Exception, e:
def test_secure(self):
zkocc_server = utils.zkocc_start()
# start the tablets
shard_0_master.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
shard_0_slave.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem",
repl_extra_flags={
'flags': "2048",
'ssl-ca': cert_dir + "/ca-cert.pem",
'ssl-cert':
cert_dir + "/client-cert.pem",
'ssl-key':
cert_dir + "/client-key.pem",
})
# Reparent using SSL
for t in [shard_0_master, shard_0_slave]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/0 ' +
shard_0_master.tablet_alias,
auto_log=True)
# then get the topology and check it
zkocc_client = zkocc.ZkOccConnection(
"localhost:%u" % environment.zkocc_port_base, "test_nj", 30.0)
topology.read_keyspaces(zkocc_client)
shard_0_master_addrs = topology.get_host_port_by_name(
zkocc_client, "test_keyspace.0.master:_vts")
if len(shard_0_master_addrs) != 1:
self.fail(
'topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts", got: %s'
% " ".join([
"%s:%u(%s)" % (h, p, str(e))
for (h, p, e) in shard_0_master_addrs
]))
if shard_0_master_addrs[0][2] != True:
self.fail(
'topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts" is not encrypted'
)
logging.debug(
"shard 0 master addrs: %s", " ".join([
"%s:%u(%s)" % (h, p, str(e))
for (h, p, e) in shard_0_master_addrs
]))
# make sure asking for optionally secure connections works too
auto_addrs = topology.get_host_port_by_name(
zkocc_client, "test_keyspace.0.master:_vtocc", encrypted=True)
if auto_addrs != shard_0_master_addrs:
self.fail(
'topology.get_host_port_by_name doesn\'t resolve encrypted addresses properly: %s != %s'
% (str(shard_0_master_addrs), str(auto_addrs)))
# try to connect with regular client
try:
conn = tablet3.TabletConnection(
"%s:%u" %
(shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]), "",
"test_keyspace", "0", 10.0)
conn.dial()
self.fail("No exception raised to secure port")
except dbexceptions.FatalError as e:
if not e.args[0][0].startswith('Unexpected EOF in handshake to'):
self.fail("Unexpected exception: %s" % str(e))
sconn = utils.get_vars(shard_0_master.port)["SecureConnections"]
if sconn != 0:
self.fail("unexpected conns %s" % sconn)
# connect to encrypted port
conn = tablet3.TabletConnection(
"%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
"",
"test_keyspace",
"0",
5.0,
encrypted=True)
conn.dial()
(results, rowcount, lastrowid,
fields) = conn._execute("select 1 from dual", {})
self.assertEqual(results, [
(1, ),
], 'wrong conn._execute output: %s' % str(results))
sconn = utils.get_vars(shard_0_master.port)["SecureConnections"]
if sconn != 1:
self.fail("unexpected conns %s" % sconn)
saccept = utils.get_vars(shard_0_master.port)["SecureAccepts"]
if saccept == 0:
self.fail("unexpected accepts %s" % saccept)
# trigger a time out on a secure connection, see what exception we get
try:
conn._execute("select sleep(100) from dual", {})
self.fail("No timeout exception")
except dbexceptions.TimeoutError as e:
logging.debug("Got the right exception for SSL timeout: %s",
str(e))
# start a vtgate to connect to that tablet
gate_proc, gate_port, gate_secure_port = utils.vtgate_start(
tablet_bson_encrypted=True,
cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
# try to connect to vtgate with regular client
timeout = 2.0
try:
conn = vtgatev2.connect([
"localhost:%s" % (gate_secure_port),
], timeout)
self.fail("No exception raised to VTGate secure port")
except dbexceptions.OperationalError as e:
exception_type = e.args[2]
exception_msg = str(e.args[2][0][0])
self.assertIsInstance(exception_type, dbexceptions.FatalError,
"unexpected exception type")
if not exception_msg.startswith('Unexpected EOF in handshake to'):
self.fail("Unexpected exception message: %s" % exception_msg)
sconn = utils.get_vars(gate_port)["SecureConnections"]
if sconn != 0:
self.fail("unexpected conns %s" % sconn)
# connect to vtgate with encrypted port
conn = vtgatev2.connect([
"localhost:%s" % (gate_secure_port),
],
timeout,
encrypted=True)
(results, rowcount, lastrowid, fields) = conn._execute(
"select 1 from dual", {},
"test_keyspace",
"master",
keyranges=[
keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE),
])
self.assertEqual(rowcount, 1, "want 1, got %d" % (rowcount))
self.assertEqual(len(fields), 1, "want 1, got %d" % (len(fields)))
self.assertEqual(results, [
(1, ),
], 'wrong conn._execute output: %s' % str(results))
sconn = utils.get_vars(gate_port)["SecureConnections"]
if sconn != 1:
self.fail("unexpected conns %s" % sconn)
saccept = utils.get_vars(gate_port)["SecureAccepts"]
if saccept == 0:
self.fail("unexpected accepts %s" % saccept)
# trigger a time out on a vtgate secure connection, see what exception we get
try:
conn._execute("select sleep(4) from dual", {},
"test_keyspace",
"master",
keyranges=[
keyrange.KeyRange(
keyrange_constants.NON_PARTIAL_KEYRANGE),
])
self.fail("No timeout exception")
except dbexceptions.TimeoutError as e:
logging.debug("Got the right exception for SSL timeout: %s",
str(e))
conn.close()
utils.vtgate_kill(gate_proc)
# kill everything
utils.kill_sub_process(zkocc_server)