def setUpModule():
global keyspace_env
global shard_0_master
global shard_1_master
global lookup_master
logging.debug('in setUpModule')
try:
environment.topo_server().setup()
logging.debug('Setting up tablets')
keyspace_env = keyspace_util.TestEnv()
keyspace_env.launch(
'user',
shards=['-80', '80-'],
ddls=[
create_vt_user,
create_vt_user2,
create_vt_user_extra,
create_vt_music,
create_vt_music_extra,
create_join_user,
create_join_user_extra,
create_join_name_info,
create_twopc_user,
],
rdonly_count=1, # to test SplitQuery
twopc_coordinator_address='localhost:15028', # enables 2pc
)
keyspace_env.launch(
'lookup',
ddls=[
create_vt_user_seq,
create_vt_music_seq,
create_vt_main_seq,
create_music_user_map,
create_name_user2_map,
create_main,
create_twopc_lookup,
],
twopc_coordinator_address='localhost:15028', # enables 2pc
)
shard_0_master = keyspace_env.tablet_map['user.-80.master']
shard_1_master = keyspace_env.tablet_map['user.80-.master']
lookup_master = keyspace_env.tablet_map['lookup.0.master']
utils.apply_vschema(vschema)
utils.VtGate().start(
tablets=[shard_0_master, shard_1_master, lookup_master],
extra_args=['-transaction_mode', 'TWOPC'])
utils.vtgate.wait_for_endpoints('user.-80.master', 1)
utils.vtgate.wait_for_endpoints('user.80-.master', 1)
utils.vtgate.wait_for_endpoints('lookup.0.master', 1)
except:
tearDownModule()
raise
def setUpModule():
global keyspace_env
global shard_0_master
global shard_1_master
global lookup_master
logging.debug('in setUpModule')
try:
environment.topo_server().setup()
logging.debug('Setting up tablets')
keyspace_env = keyspace_util.TestEnv()
keyspace_env.launch(
'user',
shards=['-80', '80-'],
ddls=[
create_vt_user,
create_vt_user2,
create_vt_user_extra,
create_vt_music,
create_vt_music_extra,
create_join_user,
create_join_user_extra,
create_join_name_info,
create_twopc_user,
],
rdonly_count=1, # to test SplitQuery
twopc_coordinator_address='localhost:15028', # enables 2pc
)
keyspace_env.launch(
'lookup',
ddls=[
create_vt_user_seq,
create_vt_music_seq,
create_vt_main_seq,
create_music_user_map,
create_name_user2_map,
create_main,
create_twopc_lookup,
],
twopc_coordinator_address='localhost:15028', # enables 2pc
)
shard_0_master = keyspace_env.tablet_map['user.-80.master']
shard_1_master = keyspace_env.tablet_map['user.80-.master']
lookup_master = keyspace_env.tablet_map['lookup.0.master']
utils.apply_vschema(vschema)
utils.VtGate().start(
tablets=[shard_0_master, shard_1_master, lookup_master],
extra_args=['-transaction_mode', 'TWOPC'])
utils.vtgate.wait_for_endpoints('user.-80.master', 1)
utils.vtgate.wait_for_endpoints('user.80-.master', 1)
utils.vtgate.wait_for_endpoints('lookup.0.master', 1)
except:
tearDownModule()
raise
def setUpModule():
global keyspace_env
global shard_0_master
global shard_1_master
global lookup_master
logging.debug('in setUpModule')
try:
environment.topo_server().setup()
logging.debug('Setting up tablets')
keyspace_env = keyspace_util.TestEnv()
keyspace_env.launch(
'user',
shards=['-80', '80-'],
ddls=[
create_vt_user,
create_vt_user2,
create_vt_user_extra,
create_vt_music,
create_vt_music_extra,
create_join_user,
create_join_user_extra,
create_join_name_info,
],
rdonly_count=1, # to test SplitQuery
)
keyspace_env.launch(
'lookup',
ddls=[
create_vt_user_seq,
create_vt_music_seq,
create_vt_main_seq,
create_music_user_map,
create_name_user2_map,
create_main,
],
)
shard_0_master = keyspace_env.tablet_map['user.-80.master']
shard_1_master = keyspace_env.tablet_map['user.80-.master']
lookup_master = keyspace_env.tablet_map['lookup.0.master']
utils.apply_vschema(vschema)
utils.VtGate().start(
tablets=[shard_0_master, shard_1_master, lookup_master])
utils.vtgate.wait_for_endpoints('user.-80.master', 1)
utils.vtgate.wait_for_endpoints('user.80-.master', 1)
utils.vtgate.wait_for_endpoints('lookup.0.master', 1)
except:
tearDownModule()
raise
def setUpModule():
global keyspace_env
global shard_0_master
global shard_1_master
global lookup_master
logging.debug('in setUpModule')
try:
environment.topo_server().setup()
logging.debug('Setting up tablets')
keyspace_env = keyspace_util.TestEnv()
keyspace_env.launch(
'user',
shards=['-80', '80-'],
ddls=[
create_vt_user,
create_vt_user2,
create_vt_user_extra,
create_vt_music,
create_vt_music_extra,
create_join_user,
create_join_user_extra,
create_join_name_info,
],
rdonly_count=1, # to test SplitQuery
)
keyspace_env.launch(
'lookup',
ddls=[
create_vt_user_seq,
create_vt_music_seq,
create_music_user_map,
create_name_user2_map,
],
)
shard_0_master = keyspace_env.tablet_map['user.-80.master']
shard_1_master = keyspace_env.tablet_map['user.80-.master']
lookup_master = keyspace_env.tablet_map['lookup.0.master']
utils.apply_vschema(vschema)
utils.VtGate().start(
tablets=[shard_0_master, shard_1_master, lookup_master])
utils.vtgate.wait_for_endpoints('user.-80.master', 1)
utils.vtgate.wait_for_endpoints('user.80-.master', 1)
utils.vtgate.wait_for_endpoints('lookup.0.master', 1)
except:
tearDownModule()
raise
def main():
parser = optparse.OptionParser(usage='usage: %prog [options]')
utils.add_options(parser)
(options, args) = parser.parse_args()
options.debug = True
utils.set_options(options)
env = keyspace_util.TestEnv()
try:
environment.topo_server().setup()
env.launch(
'user',
shards=['-80', '80-'],
ddls=[
'create table user(user_id bigint, name varchar(128), '
'primary key(user_id))',
'create table user_extra(user_id bigint, extra varchar(128), '
'primary key(user_id))',
'create table music(user_id bigint, music_id bigint, '
'primary key(user_id, music_id))',
'create table music_extra(music_id bigint, '
'keyspace_id bigint unsigned, primary key(music_id))',
],
)
env.launch(
'lookup',
ddls=[
'create table user_idx(user_id bigint not null auto_increment, '
'primary key(user_id))',
'create table name_user_idx(name varchar(128), user_id bigint, '
'primary key(name, user_id))',
'create table music_user_idx(music_id bigint not null '
'auto_increment, user_id bigint, primary key(music_id))',
],
)
utils.apply_vschema(vschema)
utils.VtGate().start(cache_ttl='500s')
utils.Vtctld().start()
print 'vtgate:', utils.vtgate.port
print 'vtctld:', utils.vtctld.port
utils.pause('the cluster is up, press enter to shut it down...')
finally:
env.teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
environment.topo_server().teardown()
def main():
parser = optparse.OptionParser(usage='usage: %prog [options]')
utils.add_options(parser)
(options, args) = parser.parse_args()
options.debug = True
utils.set_options(options)
env = keyspace_util.TestEnv()
try:
environment.topo_server().setup()
env.launch(
'user',
shards=['-80', '80-'],
ddls=[
'create table user(user_id bigint, name varchar(128), '
'primary key(user_id))',
'create table user_extra(user_id bigint, extra varchar(128), '
'primary key(user_id))',
'create table music(user_id bigint, music_id bigint, '
'primary key(user_id, music_id))',
'create table music_extra(music_id bigint, '
'keyspace_id bigint unsigned, primary key(music_id))',
],
)
env.launch(
'lookup',
ddls=[
'create table user_idx(user_id bigint not null auto_increment, '
'primary key(user_id))',
'create table name_user_idx(name varchar(128), user_id bigint, '
'primary key(name, user_id))',
'create table music_user_idx(music_id bigint not null '
'auto_increment, user_id bigint, primary key(music_id))',
],
)
utils.apply_vschema(vschema)
utils.VtGate().start(cache_ttl='500s')
utils.Vtctld().start()
print 'vtgate:', utils.vtgate.port
print 'vtctld:', utils.vtctld.port
utils.pause('the cluster is up, press enter to shut it down...')
finally:
env.teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
environment.topo_server().teardown()
def setUpModule():
global keyspace_env
global shard_0_master
global shard_1_master
global lookup_master
logging.debug('in setUpModule')
try:
environment.topo_server().setup()
logging.debug('Setting up tablets')
keyspace_env = keyspace_util.TestEnv()
keyspace_env.launch(
'user',
shards=['-80', '80-'],
ddls=[
create_vt_user,
create_vt_user2,
create_vt_user_extra,
create_vt_music,
create_vt_music_extra,
create_join_user,
create_join_user_extra,
],
)
keyspace_env.launch(
'lookup',
ddls=[
create_vt_user_seq,
create_vt_music_seq,
create_music_user_map,
create_name_user2_map,
],
)
shard_0_master = keyspace_env.tablet_map['user.-80.master']
shard_1_master = keyspace_env.tablet_map['user.80-.master']
lookup_master = keyspace_env.tablet_map['lookup.0.master']
utils.apply_vschema(vschema)
utils.VtGate().start()
except:
tearDownModule()
raise
def setUpModule():
global keyspace_env
global shard_0_master
global shard_0_replica
global shard_1_master
global lookup_master
logging.debug('in setUpModule')
try:
environment.topo_server().setup()
logging.debug('Setting up tablets')
keyspace_env = keyspace_util.TestEnv()
keyspace_env.launch(
'user',
shards=['-80', '80-'],
ddls=[
create_sharded_message,
],
)
keyspace_env.launch(
'lookup',
ddls=[
create_unsharded_message,
],
)
shard_0_master = keyspace_env.tablet_map['user.-80.master']
shard_0_replica = keyspace_env.tablet_map['user.-80.replica.0']
shard_1_master = keyspace_env.tablet_map['user.80-.master']
lookup_master = keyspace_env.tablet_map['lookup.0.master']
utils.apply_vschema(vschema)
utils.VtGate().start(tablets=[
shard_0_master, shard_0_replica, shard_1_master, lookup_master
])
utils.vtgate.wait_for_endpoints('user.-80.master', 1)
utils.vtgate.wait_for_endpoints('user.-80.replica', 1)
utils.vtgate.wait_for_endpoints('user.80-.master', 1)
utils.vtgate.wait_for_endpoints('lookup.0.master', 1)
except:
tearDownModule()
raise
def setUpModule():
global keyspace_env
global shard_0_master
global shard_1_master
global lookup_master
global vtgate_server
global vtgate_port
logging.debug("in setUpModule")
try:
environment.topo_server().setup()
logging.debug("Setting up tablets")
keyspace_env = keyspace_util.TestEnv()
keyspace_env.launch(
"user",
shards=["-80", "80-"],
ddls=[
create_vt_user,
create_vt_user2,
create_vt_user_extra,
create_vt_music,
create_vt_music_extra,
],
)
keyspace_env.launch(
"lookup",
ddls=[
create_vt_user_idx,
create_music_user_map,
create_name_user2_map,
],
)
shard_0_master = keyspace_env.tablet_map["user.-80.master"]
shard_1_master = keyspace_env.tablet_map["user.80-.master"]
lookup_master = keyspace_env.tablet_map["lookup.0.master"]
utils.apply_vschema(schema)
vtgate_server, vtgate_port = utils.vtgate_start()
except:
tearDownModule()
raise
def setUpModule():
global keyspace_env
global shard_0_master
global shard_1_master
global lookup_master
global vtgate_server
global vtgate_port
logging.debug("in setUpModule")
try:
environment.topo_server().setup()
logging.debug("Setting up tablets")
keyspace_env = keyspace_util.TestEnv()
keyspace_env.launch(
"user",
shards=["-80", "80-"],
ddls=[
create_vt_user,
create_vt_user2,
create_vt_user_extra,
create_vt_music,
create_vt_music_extra,
],
)
keyspace_env.launch(
"lookup",
ddls=[
create_vt_user_idx,
create_music_user_map,
create_name_user2_map,
],
)
shard_0_master = keyspace_env.tablet_map["user.-80.master"]
shard_1_master = keyspace_env.tablet_map["user.80-.master"]
lookup_master = keyspace_env.tablet_map["lookup.0.master"]
utils.apply_vschema(schema)
vtgate_server, vtgate_port = utils.vtgate_start()
except:
tearDownModule()
raise
def main():
parser = optparse.OptionParser(usage="usage: %prog [options]")
utils.add_options(parser)
(options, args) = parser.parse_args()
options.debug = True
utils.set_options(options)
env = keyspace_util.TestEnv()
vtgate_server = None
try:
environment.topo_server().setup()
env.launch(
"user",
shards=["-80", "80-"],
ddls=[
'create table user(user_id bigint, name varchar(128), primary key(user_id))',
'create table user_extra(user_id bigint, extra varchar(128), primary key(user_id))',
'create table music(user_id bigint, music_id bigint, primary key(user_id, music_id))',
'create table music_extra(music_id bigint, keyspace_id bigint unsigned, primary key(music_id))',
],
)
env.launch(
"lookup",
ddls=[
'create table user_idx(user_id bigint not null auto_increment, primary key(user_id))',
'create table name_user_idx(name varchar(128), user_id bigint, primary key(name, user_id))',
'create table music_user_idx(music_id bigint not null auto_increment, user_id bigint, primary key(music_id))',
],
)
utils.apply_vschema(vschema)
vtgate_server, vtgate_port = utils.vtgate_start(cache_ttl='500s')
utils.Vtctld().start()
print "vtgate:", vtgate_port
print "vtctld:", utils.vtctld.port
utils.pause("the cluster is up, press enter to shut it down...")
finally:
utils.vtgate_kill(vtgate_server)
env.teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
environment.topo_server().teardown()
def setUpModule():
global keyspace_env
global shard_0_master
global shard_0_replica
global shard_1_master
global lookup_master
logging.debug('in setUpModule')
try:
environment.topo_server().setup()
logging.debug('Setting up tablets')
keyspace_env = keyspace_util.TestEnv()
keyspace_env.launch(
'user',
shards=['-80', '80-'],
ddls=[
create_sharded_message,
],
)
keyspace_env.launch(
'lookup',
ddls=[
create_unsharded_message,
],
)
shard_0_master = keyspace_env.tablet_map['user.-80.master']
shard_0_replica = keyspace_env.tablet_map['user.-80.replica.0']
shard_1_master = keyspace_env.tablet_map['user.80-.master']
lookup_master = keyspace_env.tablet_map['lookup.0.master']
utils.apply_vschema(vschema)
utils.VtGate().start(
tablets=[shard_0_master, shard_0_replica, shard_1_master, lookup_master])
utils.vtgate.wait_for_endpoints('user.-80.master', 1)
utils.vtgate.wait_for_endpoints('user.-80.replica', 1)
utils.vtgate.wait_for_endpoints('user.80-.master', 1)
utils.vtgate.wait_for_endpoints('lookup.0.master', 1)
except:
tearDownModule()
raise
def main():
parser = optparse.OptionParser(usage="usage: %prog [options]")
utils.add_options(parser)
(options, args) = parser.parse_args()
options.debug = True
utils.set_options(options)
env = keyspace_util.TestEnv()
vtgate_server=None
try:
environment.topo_server().setup()
env.launch(
"user",
shards=["-80", "80-"],
ddls=[
'create table user(user_id bigint, name varchar(128), primary key(user_id))',
'create table user_extra(user_id bigint, extra varchar(128), primary key(user_id))',
'create table music(user_id bigint, music_id bigint, primary key(user_id, music_id))',
'create table music_extra(music_id bigint, keyspace_id bigint unsigned, primary key(music_id))',
],
)
env.launch(
"lookup",
ddls=[
'create table user_idx(user_id bigint not null auto_increment, primary key(user_id))',
'create table name_user_idx(name varchar(128), user_id bigint, primary key(name, user_id))',
'create table music_user_idx(music_id bigint not null auto_increment, user_id bigint, primary key(music_id))',
],
)
utils.apply_vschema(vschema)
vtgate_server, vtgate_port = utils.vtgate_start(cache_ttl='500s')
utils.Vtctld().start()
print "vtgate:", vtgate_port
print "vtctld:", utils.vtctld.port
utils.pause("the cluster is up, press enter to shut it down...")
finally:
utils.vtgate_kill(vtgate_server)
env.teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
environment.topo_server().teardown()
def setUpModule():
global keyspace_env
global shard_0_master
global shard_1_master
global lookup_master
logging.debug('in setUpModule')
try:
environment.topo_server().setup()
logging.debug('Setting up tablets')
keyspace_env = keyspace_util.TestEnv()
keyspace_env.launch(
'user',
shards=['-80', '80-'],
ddls=[
create_vt_user,
create_vt_user2,
create_vt_user_extra,
create_vt_music,
create_vt_music_extra,
],
)
keyspace_env.launch(
'lookup',
ddls=[
create_vt_user_idx,
create_music_user_map,
create_name_user2_map,
],
)
shard_0_master = keyspace_env.tablet_map['user.-80.master']
shard_1_master = keyspace_env.tablet_map['user.80-.master']
lookup_master = keyspace_env.tablet_map['lookup.0.master']
utils.apply_vschema(schema)
utils.VtGate().start()
except:
tearDownModule()
raise
def test_cells_aliases(self):
utils.run_vtctl(['CreateKeyspace',
'--sharding_column_name', 'custom_ksid_col',
'--sharding_column_type', base_sharding.keyspace_id_type,
'test_keyspace'])
shard_0_master.init_tablet('replica', 'test_keyspace', '-80')
shard_0_replica.init_tablet('replica', 'test_keyspace', '-80')
shard_0_rdonly.init_tablet('rdonly', 'test_keyspace', '-80')
shard_1_master.init_tablet('replica', 'test_keyspace', '80-')
shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')
shard_1_rdonly.init_tablet('rdonly', 'test_keyspace', '80-')
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
ks = utils.run_vtctl_json(['GetSrvKeyspace', 'test_nj', 'test_keyspace'])
self.assertEqual(ks['sharding_column_name'], 'custom_ksid_col')
# we set full_mycnf_args to True as a test in the KIT_BYTES case
full_mycnf_args = (base_sharding.keyspace_id_type ==
keyrange_constants.KIT_BYTES)
# create databases so vttablet can start behaving somewhat normally
for t in [shard_0_master, shard_0_replica, shard_0_rdonly,
shard_1_master, shard_1_replica, shard_1_rdonly]:
t.create_db('vt_test_keyspace')
t.start_vttablet(wait_for_state=None, full_mycnf_args=full_mycnf_args,
binlog_use_v3_resharding_mode=False)
# wait for the tablets (replication is not setup, they won't be healthy)
for t in [shard_0_master, shard_0_replica, shard_0_rdonly,
shard_1_master, shard_1_replica, shard_1_rdonly]:
t.wait_for_vttablet_state('NOT_SERVING')
# reparent to make the tablets work
utils.run_vtctl(['InitShardMaster', '-force', 'test_keyspace/-80',
shard_0_master.tablet_alias], auto_log=True)
utils.run_vtctl(['InitShardMaster', '-force', 'test_keyspace/80-',
shard_1_master.tablet_alias], auto_log=True)
# check the shards
shards = utils.run_vtctl_json(['FindAllShardsInKeyspace', 'test_keyspace'])
self.assertIn('-80', shards, 'unexpected shards: %s' % str(shards))
self.assertIn('80-', shards, 'unexpected shards: %s' % str(shards))
self.assertEqual(len(shards), 2, 'unexpected shards: %s' % str(shards))
# create the tables
self._create_schema()
self._insert_startup_values()
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
# Make sure srv keyspace graph looks as expected
utils.check_srv_keyspace(
'test_nj', 'test_keyspace',
'Partitions(master): -80 80-\n'
'Partitions(rdonly): -80 80-\n'
'Partitions(replica): -80 80-\n',
keyspace_id_type=base_sharding.keyspace_id_type,
sharding_column_name='custom_ksid_col')
utils.check_srv_keyspace(
'test_ny', 'test_keyspace',
'Partitions(master): -80 80-\n'
'Partitions(rdonly): -80 80-\n'
'Partitions(replica): -80 80-\n',
keyspace_id_type=base_sharding.keyspace_id_type,
sharding_column_name='custom_ksid_col')
# Bootstrap vtgate
utils.apply_vschema(vschema)
# Adds alias so vtgate can route to replica/rdonly tablets that are not in the same cell, but same alias
if use_alias:
utils.run_vtctl(['AddCellsAlias', '-cells', 'test_nj,test_ny','region_east_coast'], auto_log=True)
tablet_types_to_wait='MASTER,REPLICA'
else:
tablet_types_to_wait='MASTER'
utils.VtGate().start(
tablets=[shard_0_master, shard_1_master],
tablet_types_to_wait=tablet_types_to_wait,
cells_to_watch='test_nj,test_ny',
)
utils.vtgate.wait_for_endpoints('test_keyspace.-80.master', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.80-.master', 1)
vtgate_conn = self._get_connection()
result = self._execute_on_tablet_type(
vtgate_conn,
'master',
'select count(*) from test_table', {})
self.assertEqual(
result,
([(3,)], 1, 0,
[('count(*)', self.int_type)]))
if use_alias:
vtgate_conn = self._get_connection()
result = self._execute_on_tablet_type(
vtgate_conn,
'master',
'select count(*) from test_table', {})
self.assertEqual(
result,
([(3,)], 1, 0,
[('count(*)', self.int_type)]))
vtgate_conn = self._get_connection()
result = self._execute_on_tablet_type(
vtgate_conn,
'replica',
'select count(*) from test_table', {})
self.assertEqual(
result,
([(3,)], 1, 0,
[('count(*)', self.int_type)]))
vtgate_conn = self._get_connection()
result = self._execute_on_tablet_type(
vtgate_conn,
'rdonly',
'select count(*) from test_table', {})
self.assertEqual(
result,
([(3,)], 1, 0,
[('count(*)', self.int_type)]))
else:
vtgate_conn = self._get_connection()
try:
self._execute_on_tablet_type(
vtgate_conn,
'replica',
'select count(*) from test_table', {})
self.fail('Expected execute to fail, did not get error')
except Exception as e:
s = str(e)
self.assertIn('80.replica, no valid tablet: node', s)
vtgate_conn = self._get_connection()
try:
self._execute_on_tablet_type(
vtgate_conn,
'rdonly',
'select count(*) from test_table', {})
self.fail('Expected execute to fail, did not get error')
except Exception as e:
s = str(e)
self.assertIn('80.rdonly, no valid tablet: node', s)