def main(cmdline_options):
topology = vttest_pb2.VTTestTopology()
if cmdline_options.proto_topo:
# Text-encoded proto topology object, just parse it.
topology = text_format.Parse(cmdline_options.proto_topo, topology)
if not topology.cells:
topology.cells.append('test')
else:
cells = []
keyspaces = []
shard_counts = []
if cmdline_options.cells:
cells = cmdline_options.cells.split(',')
if cmdline_options.keyspaces:
keyspaces = cmdline_options.keyspaces.split(',')
if cmdline_options.num_shards:
shard_counts = [
int(x) for x in cmdline_options.num_shards.split(',')
]
for cell in cells:
topology.cells.append(cell)
for keyspace, num_shards in zip(keyspaces, shard_counts):
ks = topology.keyspaces.add(name=keyspace)
for shard in sharding_utils.get_shard_names(num_shards):
ks.shards.add(name=shard)
ks.replica_count = cmdline_options.replica_count
ks.rdonly_count = cmdline_options.rdonly_count
environment.base_port = cmdline_options.port
init_data_opts = None
if cmdline_options.initialize_with_random_data:
init_data_opts = init_data_options.InitDataOptions()
init_data_opts.rng_seed = cmdline_options.rng_seed
init_data_opts.min_table_shard_size = cmdline_options.min_table_shard_size
init_data_opts.max_table_shard_size = cmdline_options.max_table_shard_size
init_data_opts.null_probability = cmdline_options.null_probability
with local_database.LocalDatabase(
topology,
cmdline_options.schema_dir,
cmdline_options.mysql_only,
init_data_opts,
web_dir=cmdline_options.web_dir,
web_dir2=cmdline_options.web_dir2,
default_schema_dir=cmdline_options.default_schema_dir,
extra_my_cnf=os.path.join(os.environ['VTTOP'],
'config/mycnf/vtcombo.cnf')) as local_db:
print json.dumps(local_db.config())
sys.stdout.flush()
try:
raw_input()
except EOFError:
sys.stderr.write(
'WARNING: %s: No empty line was received on stdin.'
' Instead, stdin was closed and the cluster will be shut down now.'
' Make sure to send the empty line instead to proactively shutdown'
' the local cluster. For example, did you forget the shutdown in'
' your test\'s tearDown()?\n' % os.path.basename(__file__))
def setUpClass(cls):
"""Set up two keyspaces: one unsharded, one with two shards."""
topology = vttest_pb2.VTTestTopology()
topology.cells.append('test')
topology.cells.append('test2')
keyspace = topology.keyspaces.add(name='test_keyspace')
keyspace.replica_count = 2
keyspace.rdonly_count = 2
keyspace.shards.add(name='-80')
keyspace.shards.add(name='80-')
keyspace2 = topology.keyspaces.add(name='test_keyspace2')
keyspace2.shards.add(name='0')
keyspace2.replica_count = 2
keyspace2.rdonly_count = 1
cls.driver = environment.create_webdriver()
port = environment.reserve_ports(1)
vttest_environment.base_port = port
environment.reset_mysql_flavor()
cls.db = local_database.LocalDatabase(
topology,
os.path.join(environment.vttop, 'test/vttest_schema'),
False, None,
web_dir=os.path.join(environment.vttop, 'web/vtctld'),
default_schema_dir=os.path.join(
environment.vttop, 'test/vttest_schema/default'),
web_dir2=os.path.join(environment.vttop, 'web/vtctld2/app'))
cls.db.setup()
cls.vtctld_addr = 'http://localhost:%d' % cls.db.config()['port']
utils.pause('Paused test after vtcombo was started.\n'
'For manual testing, connect to vtctld: %s' % cls.vtctld_addr)
def main(cmdline_options):
topology = vttest_pb2.VTTestTopology()
if cmdline_options.proto_topo:
# Text-encoded proto topology object, just parse it.
topology = text_format.Parse(cmdline_options.proto_topo, topology)
environment.base_port = cmdline_options.port
init_data_opts = None
if cmdline_options.initialize_with_random_data:
init_data_opts = init_data_options.InitDataOptions()
init_data_opts.rng_seed = cmdline_options.rng_seed
init_data_opts.min_table_shard_size = cmdline_options.min_table_shard_size
init_data_opts.max_table_shard_size = cmdline_options.max_table_shard_size
init_data_opts.null_probability = cmdline_options.null_probability
with local_database.LocalDatabase(
topology,
cmdline_options.schema_dir,
cmdline_options.vschema,
cmdline_options.mysql_only,
init_data_opts,
web_dir=cmdline_options.web_dir) as local_db:
print json.dumps(local_db.config())
sys.stdout.flush()
try:
raw_input()
except EOFError:
sys.stderr.write(
'WARNING: %s: No empty line was received on stdin.'
' Instead, stdin was closed and the cluster will be shut down now.'
' Make sure to send the empty line instead to proactively shutdown'
' the local cluster. For example, did you forget the shutdown in'
' your test\'s tearDown()?\n' % os.path.basename(__file__))
def main(cmdline_options):
topology = vttest_pb2.VTTestTopology()
if cmdline_options.topology:
# old style topology, will disappear soon. Build a new style
# topology from it.
keyspaces = {}
for shard in cmdline_options.topology.split(','):
m = shard_exp.match(shard)
if not m:
sys.stderr.write('invalid --shard flag format: %s\n' % shard)
sys.exit(1)
keyspace = m.group(1)
shard_name = m.group(2)
db_name = m.group(3)
if keyspace not in keyspaces:
kpb = topology.keyspaces.add(name=keyspace)
keyspaces[keyspace] = kpb
keyspaces[keyspace].shards.add(name=shard_name,
db_name_override=db_name)
elif cmdline_options.proto_topo:
# new style topology, just parse it as text
topology = text_format.Parse(cmdline_options.proto_topo, topology)
environment.base_port = cmdline_options.port
init_data_opts = None
if cmdline_options.initialize_with_random_data:
init_data_opts = init_data_options.InitDataOptions()
init_data_opts.rng_seed = cmdline_options.rng_seed
init_data_opts.min_table_shard_size = cmdline_options.min_table_shard_size
init_data_opts.max_table_shard_size = cmdline_options.max_table_shard_size
init_data_opts.null_probability = cmdline_options.null_probability
with local_database.LocalDatabase(
topology,
cmdline_options.schema_dir,
cmdline_options.vschema,
cmdline_options.mysql_only,
init_data_opts,
web_dir=cmdline_options.web_dir) as local_db:
print json.dumps(local_db.config())
sys.stdout.flush()
try:
raw_input()
except EOFError:
sys.stderr.write(
'WARNING: %s: No empty line was received on stdin.'
' Instead, stdin was closed and the cluster will be shut down now.'
' Make sure to send the empty line instead to proactively shutdown'
' the local cluster. For example, did you forget the shutdown in'
' your test\'s tearDown()?\n' % os.path.basename(__file__))
def setUpClass(cls):
"""Set up two keyspaces: one unsharded, one with two shards."""
topology = vttest_pb2.VTTestTopology()
topology.cells.append('test')
topology.cells.append('test2')
keyspace = topology.keyspaces.add(name='test_keyspace')
keyspace.replica_count = 2
keyspace.rdonly_count = 2
keyspace.shards.add(name='-80')
keyspace.shards.add(name='80-')
keyspace2 = topology.keyspaces.add(name='test_keyspace2')
keyspace2.shards.add(name='0')
keyspace2.replica_count = 2
keyspace2.rdonly_count = 1
if os.environ.get('CI') == 'true' and os.environ.get(
'TRAVIS') == 'true':
username = os.environ['SAUCE_USERNAME']
access_key = os.environ['SAUCE_ACCESS_KEY']
capabilities = {}
capabilities['tunnel-identifier'] = os.environ['TRAVIS_JOB_NUMBER']
capabilities['build'] = os.environ['TRAVIS_BUILD_NUMBER']
capabilities['platform'] = 'Linux'
capabilities['browserName'] = 'chrome'
hub_url = '%s:%s@localhost:4445' % (username, access_key)
cls.driver = webdriver.Remote(desired_capabilities=capabilities,
command_executor='http://%s/wd/hub' %
hub_url)
else:
os.environ['webdriver.chrome.driver'] = os.path.join(
environment.vtroot, 'dist')
# Only testing against Chrome for now
cls.driver = webdriver.Chrome()
cls.driver.set_window_position(0, 0)
cls.driver.set_window_size(1920, 1280)
port = environment.reserve_ports(1)
vttest_environment.base_port = port
mysql_flavor.set_mysql_flavor(None)
cls.db = local_database.LocalDatabase(
topology,
os.path.join(environment.vttop, 'test/vttest_schema'),
False,
None,
web_dir=os.path.join(environment.vttop, 'web/vtctld'),
default_schema_dir=os.path.join(environment.vttop,
'test/vttest_schema/default'),
web_dir2=os.path.join(environment.vttop, 'web/vtctld2/app'))
cls.db.setup()
cls.vtctld_addr = 'http://localhost:%d' % cls.db.config()['port']
utils.pause('Paused test after vtcombo was started.\n'
'For manual testing, connect to vtctld: %s' %
cls.vtctld_addr)
def start_vitess():
"""This is the main start function."""
topology = vttest_pb2.VTTestTopology()
keyspace = topology.keyspaces.add(name='user')
keyspace.shards.add(name='-80')
keyspace.shards.add(name='80-')
keyspace = topology.keyspaces.add(name='lookup')
keyspace.shards.add(name='0')
vttop = os.environ['VTTOP']
args = [
os.path.join(vttop, 'py/vttest/run_local_database.py'), '--port',
'12345', '--proto_topo',
text_format.MessageToString(topology, as_one_line=True), '--web_dir',
os.path.join(vttop, 'web/vtctld'), '--schema_dir', 'schema'
]
sp = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
# This load will make us wait for vitess to come up.
json.loads(sp.stdout.readline())
return sp
def test_standalone(self):
"""Sample test for run_local_database.py as a standalone process."""
topology = vttest_pb2.VTTestTopology()
keyspace = topology.keyspaces.add(name='test_keyspace')
keyspace.replica_count = 2
keyspace.rdonly_count = 1
keyspace.shards.add(name='-80')
keyspace.shards.add(name='80-')
topology.keyspaces.add(name='redirect', served_from='test_keyspace')
# launch a backend database based on the provided topology and schema
port = environment.reserve_ports(1)
args = [
environment.run_local_database,
'--port',
str(port),
'--proto_topo',
text_format.MessageToString(topology, as_one_line=True),
'--schema_dir',
os.path.join(environment.vttop, 'test', 'vttest_schema'),
'--web_dir',
environment.vttop + '/web/vtctld',
]
sp = subprocess.Popen(args,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
config = json.loads(sp.stdout.readline())
# gather the vars for the vtgate process
url = 'http://localhost:%d/debug/vars' % config['port']
f = urllib.urlopen(url)
data = f.read()
f.close()
json_vars = json.loads(data)
self.assertIn('vtcombo', json_vars['cmdline'][0])
# build the vtcombo address and protocol
protocol = protocols_flavor().vttest_protocol()
if protocol == 'grpc':
vtgate_addr = 'localhost:%d' % config['grpc_port']
else:
vtgate_addr = 'localhost:%d' % config['port']
conn_timeout = 30.0
utils.pause('Paused test after vtcombo was started.\n'
'For manual testing, connect to vtgate at: %s '
'using protocol: %s.\n'
'Press enter to continue.' % (vtgate_addr, protocol))
# Remember the current timestamp after we sleep for a bit, so we
# can use it for UpdateStream later.
time.sleep(2)
before_insert = long(time.time())
# Connect to vtgate.
conn = vtgate_client.connect(protocol, vtgate_addr, conn_timeout)
# Insert a row.
row_id = 123
keyspace_id = get_keyspace_id(row_id)
cursor = conn.cursor(tablet_type='master',
keyspace='test_keyspace',
keyspace_ids=[pack_kid(keyspace_id)],
writable=True)
cursor.begin()
insert = ('insert into test_table (id, msg, keyspace_id) values (:id, '
':msg, :keyspace_id)')
bind_variables = {
'id': row_id,
'msg': 'test %s' % row_id,
'keyspace_id': keyspace_id,
}
cursor.execute(insert, bind_variables)
cursor.commit()
# Read the row back.
cursor.execute('select * from test_table where id=:id', {'id': row_id})
result = cursor.fetchall()
self.assertEqual(result[0][1], 'test 123')
# try to insert again, see if we get the right integrity error exception
# (this is meant to test vtcombo properly returns exceptions, and to a
# lesser extent that the python client converts it properly)
cursor.begin()
with self.assertRaises(dbexceptions.IntegrityError):
cursor.execute(insert, bind_variables)
cursor.rollback()
# Insert a bunch of rows with long msg values.
bind_variables['msg'] = 'x' * 64
id_start = 1000
rowcount = 500
cursor.begin()
for i in xrange(id_start, id_start + rowcount):
bind_variables['id'] = i
bind_variables['keyspace_id'] = get_keyspace_id(i)
cursor.execute(insert, bind_variables)
cursor.commit()
cursor.close()
# Try to fetch a large number of rows, from a rdonly
# (more than one streaming result packet).
stream_cursor = conn.cursor(
tablet_type='rdonly',
keyspace='test_keyspace',
keyspace_ids=[pack_kid(keyspace_id)],
cursorclass=vtgate_cursor.StreamVTGateCursor)
stream_cursor.execute('select * from test_table where id >= :id_start',
{'id_start': id_start})
self.assertEqual(rowcount, len(list(stream_cursor.fetchall())))
stream_cursor.close()
# try to read a row using the redirected keyspace, to a replica this time
row_id = 123
keyspace_id = get_keyspace_id(row_id)
cursor = conn.cursor(tablet_type='replica',
keyspace='redirect',
keyspace_ids=[pack_kid(keyspace_id)])
cursor.execute('select * from test_table where id=:id', {'id': row_id})
result = cursor.fetchall()
self.assertEqual(result[0][1], 'test 123')
cursor.close()
# Try to get the update stream from the connection. This makes
# sure that part works as well.
count = 0
for (event, _) in conn.update_stream('test_keyspace',
topodata_pb2.MASTER,
timestamp=before_insert,
shard='-80'):
for statement in event.statements:
if statement.table_name == 'test_table':
count += 1
if count == rowcount + 1:
# We're getting the initial value, plus the 500 updates.
break
# Insert a sentinel value into the second shard.
row_id = 0x8100000000000000
keyspace_id = get_keyspace_id(row_id)
cursor = conn.cursor(tablet_type='master',
keyspace='test_keyspace',
keyspace_ids=[pack_kid(keyspace_id)],
writable=True)
cursor.begin()
bind_variables = {
'id': row_id,
'msg': 'test %s' % row_id,
'keyspace_id': keyspace_id,
}
cursor.execute(insert, bind_variables)
cursor.commit()
cursor.close()
# Try to connect to an update stream on the other shard.
# We may get some random update stream events, but we should not get any
# event that's related to the first shard. Only events related to
# the Insert we just did.
found = False
for (event, _) in conn.update_stream('test_keyspace',
topodata_pb2.MASTER,
timestamp=before_insert,
shard='80-'):
for statement in event.statements:
self.assertEqual(statement.table_name, 'test_table')
fields, rows = proto3_encoding.convert_stream_event_statement(
statement)
self.assertEqual(fields[0], 'id')
self.assertEqual(rows[0][0], row_id)
found = True
if found:
break
# Clean up the connection
conn.close()
# Test we can connect to vtcombo for vtctl actions
protocol = protocols_flavor().vtctl_python_client_protocol()
if protocol == 'grpc':
vtgate_addr = 'localhost:%d' % config['grpc_port']
else:
vtgate_addr = 'localhost:%d' % config['port']
out, _ = utils.run(environment.binary_args('vtctlclient') + [
'-vtctl_client_protocol',
protocol,
'-server',
vtgate_addr,
'-stderrthreshold',
'0',
'ListAllTablets',
'test',
],
trap_output=True)
num_master = 0
num_replica = 0
num_rdonly = 0
num_dash_80 = 0
num_80_dash = 0
for line in out.splitlines():
parts = line.split()
self.assertEqual(parts[1], 'test_keyspace',
'invalid keyspace in line: %s' % line)
if parts[3] == 'master':
num_master += 1
elif parts[3] == 'replica':
num_replica += 1
elif parts[3] == 'rdonly':
num_rdonly += 1
else:
self.fail('invalid tablet type in line: %s' % line)
if parts[2] == '-80':
num_dash_80 += 1
elif parts[2] == '80-':
num_80_dash += 1
else:
self.fail('invalid shard name in line: %s' % line)
self.assertEqual(num_master, 2)
self.assertEqual(num_replica, 2)
self.assertEqual(num_rdonly, 2)
self.assertEqual(num_dash_80, 3)
self.assertEqual(num_80_dash, 3)
# and we're done, clean-up process
sp.stdin.write('\n')
sp.wait()
