def main(cmdline_options): topology = vttest_pb2.VTTestTopology() if cmdline_options.proto_topo: # Text-encoded proto topology object, just parse it. topology = text_format.Parse(cmdline_options.proto_topo, topology) if not topology.cells: topology.cells.append('test') else: cells = [] keyspaces = [] shard_counts = [] if cmdline_options.cells: cells = cmdline_options.cells.split(',') if cmdline_options.keyspaces: keyspaces = cmdline_options.keyspaces.split(',') if cmdline_options.num_shards: shard_counts = [ int(x) for x in cmdline_options.num_shards.split(',') ] for cell in cells: topology.cells.append(cell) for keyspace, num_shards in zip(keyspaces, shard_counts): ks = topology.keyspaces.add(name=keyspace) for shard in sharding_utils.get_shard_names(num_shards): ks.shards.add(name=shard) ks.replica_count = cmdline_options.replica_count ks.rdonly_count = cmdline_options.rdonly_count environment.base_port = cmdline_options.port init_data_opts = None if cmdline_options.initialize_with_random_data: init_data_opts = init_data_options.InitDataOptions() init_data_opts.rng_seed = cmdline_options.rng_seed init_data_opts.min_table_shard_size = cmdline_options.min_table_shard_size init_data_opts.max_table_shard_size = cmdline_options.max_table_shard_size init_data_opts.null_probability = cmdline_options.null_probability with local_database.LocalDatabase( topology, cmdline_options.schema_dir, cmdline_options.mysql_only, init_data_opts, web_dir=cmdline_options.web_dir, web_dir2=cmdline_options.web_dir2, default_schema_dir=cmdline_options.default_schema_dir, extra_my_cnf=os.path.join(os.environ['VTTOP'], 'config/mycnf/vtcombo.cnf')) as local_db: print json.dumps(local_db.config()) sys.stdout.flush() try: raw_input() except EOFError: sys.stderr.write( 'WARNING: %s: No empty line was received on stdin.' ' Instead, stdin was closed and the cluster will be shut down now.' ' Make sure to send the empty line instead to proactively shutdown' ' the local cluster. For example, did you forget the shutdown in' ' your test\'s tearDown()?\n' % os.path.basename(__file__))
def setUpClass(cls): """Set up two keyspaces: one unsharded, one with two shards.""" topology = vttest_pb2.VTTestTopology() topology.cells.append('test') topology.cells.append('test2') keyspace = topology.keyspaces.add(name='test_keyspace') keyspace.replica_count = 2 keyspace.rdonly_count = 2 keyspace.shards.add(name='-80') keyspace.shards.add(name='80-') keyspace2 = topology.keyspaces.add(name='test_keyspace2') keyspace2.shards.add(name='0') keyspace2.replica_count = 2 keyspace2.rdonly_count = 1 cls.driver = environment.create_webdriver() port = environment.reserve_ports(1) vttest_environment.base_port = port environment.reset_mysql_flavor() cls.db = local_database.LocalDatabase( topology, os.path.join(environment.vttop, 'test/vttest_schema'), False, None, web_dir=os.path.join(environment.vttop, 'web/vtctld'), default_schema_dir=os.path.join( environment.vttop, 'test/vttest_schema/default'), web_dir2=os.path.join(environment.vttop, 'web/vtctld2/app')) cls.db.setup() cls.vtctld_addr = 'http://localhost:%d' % cls.db.config()['port'] utils.pause('Paused test after vtcombo was started.\n' 'For manual testing, connect to vtctld: %s' % cls.vtctld_addr)
def main(cmdline_options): topology = vttest_pb2.VTTestTopology() if cmdline_options.proto_topo: # Text-encoded proto topology object, just parse it. topology = text_format.Parse(cmdline_options.proto_topo, topology) environment.base_port = cmdline_options.port init_data_opts = None if cmdline_options.initialize_with_random_data: init_data_opts = init_data_options.InitDataOptions() init_data_opts.rng_seed = cmdline_options.rng_seed init_data_opts.min_table_shard_size = cmdline_options.min_table_shard_size init_data_opts.max_table_shard_size = cmdline_options.max_table_shard_size init_data_opts.null_probability = cmdline_options.null_probability with local_database.LocalDatabase( topology, cmdline_options.schema_dir, cmdline_options.vschema, cmdline_options.mysql_only, init_data_opts, web_dir=cmdline_options.web_dir) as local_db: print json.dumps(local_db.config()) sys.stdout.flush() try: raw_input() except EOFError: sys.stderr.write( 'WARNING: %s: No empty line was received on stdin.' ' Instead, stdin was closed and the cluster will be shut down now.' ' Make sure to send the empty line instead to proactively shutdown' ' the local cluster. For example, did you forget the shutdown in' ' your test\'s tearDown()?\n' % os.path.basename(__file__))
def main(cmdline_options): topology = vttest_pb2.VTTestTopology() if cmdline_options.topology: # old style topology, will disappear soon. Build a new style # topology from it. keyspaces = {} for shard in cmdline_options.topology.split(','): m = shard_exp.match(shard) if not m: sys.stderr.write('invalid --shard flag format: %s\n' % shard) sys.exit(1) keyspace = m.group(1) shard_name = m.group(2) db_name = m.group(3) if keyspace not in keyspaces: kpb = topology.keyspaces.add(name=keyspace) keyspaces[keyspace] = kpb keyspaces[keyspace].shards.add(name=shard_name, db_name_override=db_name) elif cmdline_options.proto_topo: # new style topology, just parse it as text topology = text_format.Parse(cmdline_options.proto_topo, topology) environment.base_port = cmdline_options.port init_data_opts = None if cmdline_options.initialize_with_random_data: init_data_opts = init_data_options.InitDataOptions() init_data_opts.rng_seed = cmdline_options.rng_seed init_data_opts.min_table_shard_size = cmdline_options.min_table_shard_size init_data_opts.max_table_shard_size = cmdline_options.max_table_shard_size init_data_opts.null_probability = cmdline_options.null_probability with local_database.LocalDatabase( topology, cmdline_options.schema_dir, cmdline_options.vschema, cmdline_options.mysql_only, init_data_opts, web_dir=cmdline_options.web_dir) as local_db: print json.dumps(local_db.config()) sys.stdout.flush() try: raw_input() except EOFError: sys.stderr.write( 'WARNING: %s: No empty line was received on stdin.' ' Instead, stdin was closed and the cluster will be shut down now.' ' Make sure to send the empty line instead to proactively shutdown' ' the local cluster. For example, did you forget the shutdown in' ' your test\'s tearDown()?\n' % os.path.basename(__file__))
def setUpClass(cls): """Set up two keyspaces: one unsharded, one with two shards.""" topology = vttest_pb2.VTTestTopology() topology.cells.append('test') topology.cells.append('test2') keyspace = topology.keyspaces.add(name='test_keyspace') keyspace.replica_count = 2 keyspace.rdonly_count = 2 keyspace.shards.add(name='-80') keyspace.shards.add(name='80-') keyspace2 = topology.keyspaces.add(name='test_keyspace2') keyspace2.shards.add(name='0') keyspace2.replica_count = 2 keyspace2.rdonly_count = 1 if os.environ.get('CI') == 'true' and os.environ.get( 'TRAVIS') == 'true': username = os.environ['SAUCE_USERNAME'] access_key = os.environ['SAUCE_ACCESS_KEY'] capabilities = {} capabilities['tunnel-identifier'] = os.environ['TRAVIS_JOB_NUMBER'] capabilities['build'] = os.environ['TRAVIS_BUILD_NUMBER'] capabilities['platform'] = 'Linux' capabilities['browserName'] = 'chrome' hub_url = '%s:%s@localhost:4445' % (username, access_key) cls.driver = webdriver.Remote(desired_capabilities=capabilities, command_executor='http://%s/wd/hub' % hub_url) else: os.environ['webdriver.chrome.driver'] = os.path.join( environment.vtroot, 'dist') # Only testing against Chrome for now cls.driver = webdriver.Chrome() cls.driver.set_window_position(0, 0) cls.driver.set_window_size(1920, 1280) port = environment.reserve_ports(1) vttest_environment.base_port = port mysql_flavor.set_mysql_flavor(None) cls.db = local_database.LocalDatabase( topology, os.path.join(environment.vttop, 'test/vttest_schema'), False, None, web_dir=os.path.join(environment.vttop, 'web/vtctld'), default_schema_dir=os.path.join(environment.vttop, 'test/vttest_schema/default'), web_dir2=os.path.join(environment.vttop, 'web/vtctld2/app')) cls.db.setup() cls.vtctld_addr = 'http://localhost:%d' % cls.db.config()['port'] utils.pause('Paused test after vtcombo was started.\n' 'For manual testing, connect to vtctld: %s' % cls.vtctld_addr)
def start_vitess(): """This is the main start function.""" topology = vttest_pb2.VTTestTopology() keyspace = topology.keyspaces.add(name='user') keyspace.shards.add(name='-80') keyspace.shards.add(name='80-') keyspace = topology.keyspaces.add(name='lookup') keyspace.shards.add(name='0') vttop = os.environ['VTTOP'] args = [ os.path.join(vttop, 'py/vttest/run_local_database.py'), '--port', '12345', '--proto_topo', text_format.MessageToString(topology, as_one_line=True), '--web_dir', os.path.join(vttop, 'web/vtctld'), '--schema_dir', 'schema' ] sp = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE) # This load will make us wait for vitess to come up. json.loads(sp.stdout.readline()) return sp
def test_standalone(self): """Sample test for run_local_database.py as a standalone process.""" topology = vttest_pb2.VTTestTopology() keyspace = topology.keyspaces.add(name='test_keyspace') keyspace.replica_count = 2 keyspace.rdonly_count = 1 keyspace.shards.add(name='-80') keyspace.shards.add(name='80-') topology.keyspaces.add(name='redirect', served_from='test_keyspace') # launch a backend database based on the provided topology and schema port = environment.reserve_ports(1) args = [ environment.run_local_database, '--port', str(port), '--proto_topo', text_format.MessageToString(topology, as_one_line=True), '--schema_dir', os.path.join(environment.vttop, 'test', 'vttest_schema'), '--web_dir', environment.vttop + '/web/vtctld', ] sp = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE) config = json.loads(sp.stdout.readline()) # gather the vars for the vtgate process url = 'http://localhost:%d/debug/vars' % config['port'] f = urllib.urlopen(url) data = f.read() f.close() json_vars = json.loads(data) self.assertIn('vtcombo', json_vars['cmdline'][0]) # build the vtcombo address and protocol protocol = protocols_flavor().vttest_protocol() if protocol == 'grpc': vtgate_addr = 'localhost:%d' % config['grpc_port'] else: vtgate_addr = 'localhost:%d' % config['port'] conn_timeout = 30.0 utils.pause('Paused test after vtcombo was started.\n' 'For manual testing, connect to vtgate at: %s ' 'using protocol: %s.\n' 'Press enter to continue.' % (vtgate_addr, protocol)) # Remember the current timestamp after we sleep for a bit, so we # can use it for UpdateStream later. time.sleep(2) before_insert = long(time.time()) # Connect to vtgate. conn = vtgate_client.connect(protocol, vtgate_addr, conn_timeout) # Insert a row. row_id = 123 keyspace_id = get_keyspace_id(row_id) cursor = conn.cursor(tablet_type='master', keyspace='test_keyspace', keyspace_ids=[pack_kid(keyspace_id)], writable=True) cursor.begin() insert = ('insert into test_table (id, msg, keyspace_id) values (:id, ' ':msg, :keyspace_id)') bind_variables = { 'id': row_id, 'msg': 'test %s' % row_id, 'keyspace_id': keyspace_id, } cursor.execute(insert, bind_variables) cursor.commit() # Read the row back. cursor.execute('select * from test_table where id=:id', {'id': row_id}) result = cursor.fetchall() self.assertEqual(result[0][1], 'test 123') # try to insert again, see if we get the right integrity error exception # (this is meant to test vtcombo properly returns exceptions, and to a # lesser extent that the python client converts it properly) cursor.begin() with self.assertRaises(dbexceptions.IntegrityError): cursor.execute(insert, bind_variables) cursor.rollback() # Insert a bunch of rows with long msg values. bind_variables['msg'] = 'x' * 64 id_start = 1000 rowcount = 500 cursor.begin() for i in xrange(id_start, id_start + rowcount): bind_variables['id'] = i bind_variables['keyspace_id'] = get_keyspace_id(i) cursor.execute(insert, bind_variables) cursor.commit() cursor.close() # Try to fetch a large number of rows, from a rdonly # (more than one streaming result packet). stream_cursor = conn.cursor( tablet_type='rdonly', keyspace='test_keyspace', keyspace_ids=[pack_kid(keyspace_id)], cursorclass=vtgate_cursor.StreamVTGateCursor) stream_cursor.execute('select * from test_table where id >= :id_start', {'id_start': id_start}) self.assertEqual(rowcount, len(list(stream_cursor.fetchall()))) stream_cursor.close() # try to read a row using the redirected keyspace, to a replica this time row_id = 123 keyspace_id = get_keyspace_id(row_id) cursor = conn.cursor(tablet_type='replica', keyspace='redirect', keyspace_ids=[pack_kid(keyspace_id)]) cursor.execute('select * from test_table where id=:id', {'id': row_id}) result = cursor.fetchall() self.assertEqual(result[0][1], 'test 123') cursor.close() # Try to get the update stream from the connection. This makes # sure that part works as well. count = 0 for (event, _) in conn.update_stream('test_keyspace', topodata_pb2.MASTER, timestamp=before_insert, shard='-80'): for statement in event.statements: if statement.table_name == 'test_table': count += 1 if count == rowcount + 1: # We're getting the initial value, plus the 500 updates. break # Insert a sentinel value into the second shard. row_id = 0x8100000000000000 keyspace_id = get_keyspace_id(row_id) cursor = conn.cursor(tablet_type='master', keyspace='test_keyspace', keyspace_ids=[pack_kid(keyspace_id)], writable=True) cursor.begin() bind_variables = { 'id': row_id, 'msg': 'test %s' % row_id, 'keyspace_id': keyspace_id, } cursor.execute(insert, bind_variables) cursor.commit() cursor.close() # Try to connect to an update stream on the other shard. # We may get some random update stream events, but we should not get any # event that's related to the first shard. Only events related to # the Insert we just did. found = False for (event, _) in conn.update_stream('test_keyspace', topodata_pb2.MASTER, timestamp=before_insert, shard='80-'): for statement in event.statements: self.assertEqual(statement.table_name, 'test_table') fields, rows = proto3_encoding.convert_stream_event_statement( statement) self.assertEqual(fields[0], 'id') self.assertEqual(rows[0][0], row_id) found = True if found: break # Clean up the connection conn.close() # Test we can connect to vtcombo for vtctl actions protocol = protocols_flavor().vtctl_python_client_protocol() if protocol == 'grpc': vtgate_addr = 'localhost:%d' % config['grpc_port'] else: vtgate_addr = 'localhost:%d' % config['port'] out, _ = utils.run(environment.binary_args('vtctlclient') + [ '-vtctl_client_protocol', protocol, '-server', vtgate_addr, '-stderrthreshold', '0', 'ListAllTablets', 'test', ], trap_output=True) num_master = 0 num_replica = 0 num_rdonly = 0 num_dash_80 = 0 num_80_dash = 0 for line in out.splitlines(): parts = line.split() self.assertEqual(parts[1], 'test_keyspace', 'invalid keyspace in line: %s' % line) if parts[3] == 'master': num_master += 1 elif parts[3] == 'replica': num_replica += 1 elif parts[3] == 'rdonly': num_rdonly += 1 else: self.fail('invalid tablet type in line: %s' % line) if parts[2] == '-80': num_dash_80 += 1 elif parts[2] == '80-': num_80_dash += 1 else: self.fail('invalid shard name in line: %s' % line) self.assertEqual(num_master, 2) self.assertEqual(num_replica, 2) self.assertEqual(num_rdonly, 2) self.assertEqual(num_dash_80, 3) self.assertEqual(num_80_dash, 3) # and we're done, clean-up process sp.stdin.write('\n') sp.wait()