def start_vttablet(self, port=None, auth=False, memcache=False, wait_for_state="SERVING", customrules=None, schema_override=None, cert=None, key=None, ca_cert=None, repl_extra_flags={}): """ Starts a vttablet process, and returns it. The process is also saved in self.proc, so it's easy to kill as well. """ environment.prog_compile('vtaction') args = [environment.binary_path('vttablet'), '-port', '%s' % (port or self.port), '-tablet-path', self.tablet_alias, '-log_dir', environment.vtlogroot] args.extend(environment.topo_server_flags()) args.extend(environment.binlog_player_protocol_flags()) dbconfigs = self._get_db_configs_file(repl_extra_flags) for key1 in dbconfigs: for key2 in dbconfigs[key1]: args.extend(["-db-config-"+key1+"-"+key2, dbconfigs[key1][key2]]) if memcache: if os.path.exists(environment.vtroot + "/bin/memcached"): args.extend(["-rowcache-bin", environment.vtroot + "/bin/memcached"]) else: args.extend(["-rowcache-bin", "memcached"]) memcache_socket = os.path.join(self.tablet_dir, "memcache.sock") args.extend(["-rowcache-socket", memcache_socket]) args.extend(["-enable-rowcache"]) if auth: args.extend(['-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json')]) if customrules: args.extend(['-customrules', customrules]) if schema_override: args.extend(['-schema-override', schema_override]) if cert: self.secure_port = environment.reserve_ports(1) args.extend(['-secure-port', '%s' % self.secure_port, '-cert', cert, '-key', key]) if ca_cert: args.extend(['-ca-cert', ca_cert]) stderr_fd = open(os.path.join(self.tablet_dir, "vttablet.stderr"), "w") # increment count only the first time if not self.proc: Tablet.tablets_running += 1 self.proc = utils.run_bg(args, stderr=stderr_fd) stderr_fd.close() # wait for zookeeper PID just to be sure we have it if environment.topo_server_implementation == 'zookeeper': utils.run(environment.binary_path('zk')+' wait -e ' + self.zk_pid, stdout=utils.devnull) # wait for query service to be in the right state if wait_for_state: self.wait_for_vttablet_state(wait_for_state, port=port) return self.proc
def start_vttablet(self, port=None, auth=False, memcache=False, wait_for_state="SERVING", customrules=None, schema_override=None, cert=None, key=None, ca_cert=None, repl_extra_flags={}): """ Starts a vttablet process, and returns it. The process is also saved in self.proc, so it's easy to kill as well. """ environment.prog_compile('vtaction') args = [environment.binary_path('vttablet'), '-port', '%s' % (port or self.port), '-tablet-path', self.tablet_alias, '-log_dir', environment.vtlogroot] args.extend(environment.topo_server_flags()) args.extend(utils.binlog_player_protocol_flags) dbconfigs = self._get_db_configs_file(repl_extra_flags) for key1 in dbconfigs: for key2 in dbconfigs[key1]: args.extend(["-db-config-"+key1+"-"+key2, dbconfigs[key1][key2]]) if memcache: if os.path.exists(environment.vtroot + "/bin/memcached"): args.extend(["-rowcache-bin", environment.vtroot + "/bin/memcached"]) else: args.extend(["-rowcache-bin", "memcached"]) memcache_socket = os.path.join(self.tablet_dir, "memcache.sock") args.extend(["-rowcache-socket", memcache_socket]) args.extend(["-enable-rowcache"]) if auth: args.extend(['-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json')]) if customrules: args.extend(['-customrules', customrules]) if schema_override: args.extend(['-schema-override', schema_override]) if cert: self.secure_port = environment.reserve_ports(1) args.extend(['-secure-port', '%s' % self.secure_port, '-cert', cert, '-key', key]) if ca_cert: args.extend(['-ca-cert', ca_cert]) stderr_fd = open(os.path.join(self.tablet_dir, "vttablet.stderr"), "w") # increment count only the first time if not self.proc: Tablet.tablets_running += 1 self.proc = utils.run_bg(args, stderr=stderr_fd) stderr_fd.close() # wait for zookeeper PID just to be sure we have it if environment.topo_server_implementation == 'zookeeper': utils.run(environment.binary_path('zk')+' wait -e ' + self.zk_pid, stdout=utils.devnull) # wait for query service to be in the right state if wait_for_state: self.wait_for_vttablet_state(wait_for_state, port=port) return self.proc
def test_get_srv_keyspace_names(self): utils.run_vtctl('CreateKeyspace test_keyspace1') utils.run_vtctl('CreateKeyspace test_keyspace2') t1 = tablet.Tablet(tablet_uid=1, cell="nj") t1.init_tablet("master", "test_keyspace1", "0") t1.update_addrs() t2 = tablet.Tablet(tablet_uid=2, cell="nj") t2.init_tablet("master", "test_keyspace2", "0") t2.update_addrs() utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace*'], auto_log=True) # vtgate API test out, err = utils.run(environment.binary_path('zkclient2')+' -server localhost:%u -mode getSrvKeyspaceNames test_nj' % self.vtgate_zk_port, trap_output=True) self.assertEqual(err, "KeyspaceNames[0] = test_keyspace1\n" + "KeyspaceNames[1] = test_keyspace2\n") if environment.topo_server_implementation == 'zookeeper': self.assertItemsEqual(self.topo.get_srv_keyspace_names('local'), ["test_keyspace1", "test_keyspace2"]) # zkocc API test out, err = utils.run(environment.binary_path('zkclient2')+' -server localhost:%u -mode getSrvKeyspaceNames test_nj' % environment.zkocc_port_base, trap_output=True) self.assertEqual(err, "KeyspaceNames[0] = test_keyspace1\n" + "KeyspaceNames[1] = test_keyspace2\n") # vtgate zkocc API test out, err = utils.run(environment.binary_path('zkclient2')+' -server localhost:%u -mode getSrvKeyspaceNames test_nj' % self.vtgate_zkocc_port, trap_output=True) self.assertEqual(err, "KeyspaceNames[0] = test_keyspace1\n" + "KeyspaceNames[1] = test_keyspace2\n")
def setUp(self): environment.topo_server_wipe() if environment.topo_server_implementation == 'zookeeper': utils.run( environment.binary_path('zk') + ' touch -p /zk/test_nj/vt/zkocc1') utils.run( environment.binary_path('zk') + ' touch -p /zk/test_nj/vt/zkocc2') fd = tempfile.NamedTemporaryFile( dir=environment.tmproot, delete=False) filename1 = fd.name fd.write("Test data 1") fd.close() utils.run( environment.binary_path('zk') + ' cp ' + filename1 + ' /zk/test_nj/vt/zkocc1/data1') fd = tempfile.NamedTemporaryFile( dir=environment.tmproot, delete=False) filename2 = fd.name fd.write("Test data 2") fd.close() utils.run( environment.binary_path('zk') + ' cp ' + filename2 + ' /zk/test_nj/vt/zkocc1/data2') fd = tempfile.NamedTemporaryFile( dir=environment.tmproot, delete=False) filename3 = fd.name fd.write("Test data 3") fd.close() utils.run( environment.binary_path('zk') + ' cp ' + filename3 + ' /zk/test_nj/vt/zkocc1/data3')
def _check_zk_output(self, cmd, expected): # directly for sanity out, err = utils.run(environment.binary_path('zk')+' ' + cmd, trap_output=True) self.assertEqualNormalized(out, expected, 'unexpected direct zk output') # using zkocc out, err = utils.run(environment.binary_path('zk')+' --zk.zkocc-addr=localhost:%u %s' % (environment.zkocc_port_base, cmd), trap_output=True) self.assertEqualNormalized(out, expected, 'unexpected zk zkocc output') logging.debug("Matched: %s", out)
def test_get_srv_keyspace(self): utils.run_vtctl('CreateKeyspace test_keyspace') t = tablet.Tablet(tablet_uid=1, cell="nj") t.init_tablet("master", "test_keyspace", "0") utils.run_vtctl( 'UpdateTabletAddrs -hostname localhost -ip-addr 127.0.0.1 -mysql-port %s -vts-port %s %s' % (t.mysql_port, t.port + 500, t.tablet_alias)) self.rebuild() # vtgate zk API test out, err = utils.run( environment.binary_path('zkclient2') + ' -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace' % self.vtgate_zk_port, trap_output=True) self.assertEqual( err, "Partitions[master] =\n" + " Shards[0]={Start: , End: }\n" + "Partitions[rdonly] =\n" + " Shards[0]={Start: , End: }\n" + "Partitions[replica] =\n" + " Shards[0]={Start: , End: }\n" + "Shards[0]={Start: , End: }\n" + "TabletTypes[0] = master\n", "Got wrong content: %s" % err) if environment.topo_server_implementation == 'zookeeper': reply = self.topo.get_srv_keyspace("test_nj", "test_keyspace") self.assertEqual(reply['TabletTypes'], ['master']) # zkocc API test out, err = utils.run( environment.binary_path('zkclient2') + ' -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace' % environment.zkocc_port_base, trap_output=True) self.assertEqual( err, "Partitions[master] =\n" + " Shards[0]={Start: , End: }\n" + "Partitions[rdonly] =\n" + " Shards[0]={Start: , End: }\n" + "Partitions[replica] =\n" + " Shards[0]={Start: , End: }\n" + "Shards[0]={Start: , End: }\n" + "TabletTypes[0] = master\n", "Got wrong content: %s" % err) # vtgate zkocc API test out, err = utils.run( environment.binary_path('zkclient2') + ' -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace' % self.vtgate_zkocc_port, trap_output=True) self.assertEqual( err, "Partitions[master] =\n" + " Shards[0]={Start: , End: }\n" + "Partitions[rdonly] =\n" + " Shards[0]={Start: , End: }\n" + "Partitions[replica] =\n" + " Shards[0]={Start: , End: }\n" + "Shards[0]={Start: , End: }\n" + "TabletTypes[0] = master\n", "Got wrong content: %s" % err)
def mysqlctl(self, cmd, extra_my_cnf=None, with_ports=False): all_extra_my_cnf = [] if environment.mysql_flavor == "GoogleMysql": # we have to manually enable hierarchical replication to support groupid all_extra_my_cnf.append(environment.vttop + "/config/mycnf/master_google.cnf") if extra_my_cnf: all_extra_my_cnf.append(extra_my_cnf) env = None if all_extra_my_cnf: env = os.environ.copy() env['EXTRA_MY_CNF'] = ":".join(all_extra_my_cnf) args = [ environment.binary_path('mysqlctl'), '-log_dir', environment.vtlogroot, '-tablet_uid', str(self.tablet_uid) ] if with_ports: args.extend( ['-port', str(self.port), '-mysql_port', str(self.mysql_port)]) if utils.options.verbose == 2: args.append('-alsologtostderr') args.extend(cmd) return utils.run_bg(args, env=env)
def vtclient2(uid, path, query, bindvars=None, user=None, password=None, driver=None, verbose=False, raise_on_error=True): if (user is None) != (password is None): raise TypeError("you should provide either both or none of user and password") # for ZK paths to not have // in the path, that confuses things if path.startswith('/'): path = path[1:] server = "localhost:%u/%s" % (uid, path) cmdline = [environment.binary_path('vtclient2'), '-server', server] cmdline += environment.topo_server_flags() cmdline += environment.tabletconn_protocol_flags() if user is not None: cmdline.extend(['-tablet-bson-username', user, '-tablet-bson-password', password]) if bindvars: cmdline.extend(['-bindvars', bindvars]) if driver: cmdline.extend(['-driver', driver]) if verbose: cmdline.extend(['-alsologtostderr', '-verbose']) cmdline.append(query) return run(cmdline, raise_on_error=raise_on_error, trap_output=True)
def tearDown(self): try: mcu = self.mysql_conn.cursor() for line in self.clean_sqls: try: mcu.execute(line, {}) except: pass mcu.close() except: pass if getattr(self, "txlogger", None): self.txlogger.terminate() if getattr(self, "vtocc", None): self.vtocc.terminate() # stop mysql, delete directory subprocess.call([ environment.binary_path('mysqlctl'), "-tablet-uid", self.tabletuid, "teardown", "-force" ]) try: shutil.rmtree(self.mysqldir) except: pass
def vtclient2(uid, path, query, bindvars=None, user=None, password=None, driver=None, verbose=False, raise_on_error=True): if (user is None) != (password is None): raise TypeError( "you should provide either both or none of user and password") # for ZK paths to not have // in the path, that confuses things if path.startswith('/'): path = path[1:] server = "localhost:%u/%s" % (uid, path) cmdline = [environment.binary_path('vtclient2'), '-server', server] cmdline += environment.topo_server_flags() cmdline += environment.tabletconn_protocol_flags() if user is not None: cmdline.extend( ['-tablet-bson-username', user, '-tablet-bson-password', password]) if bindvars: cmdline.extend(['-bindvars', bindvars]) if driver: cmdline.extend(['-driver', driver]) if verbose: cmdline.extend(['-alsologtostderr', '-verbose']) cmdline.append(query) return run(cmdline, raise_on_error=raise_on_error, trap_output=True)
def test_sigterm(self): utils.run_vtctl('CreateKeyspace test_keyspace') # create the database so vttablets start, as it is serving tablet_62344.create_db('vt_test_keyspace') tablet_62344.init_tablet('master', 'test_keyspace', '0', start=True) # start a 'vtctl Sleep' command in the background args = [ environment.binary_path('vtctl'), '-log_dir', environment.vtlogroot, '--alsologtostderr' ] args.extend(environment.topo_server_flags()) args.extend(environment.tablet_manager_protocol_flags()) args.extend(['Sleep', tablet_62344.tablet_alias, '60s']) sp = utils.run_bg(args, stdout=PIPE, stderr=PIPE) # wait for it to start, and let's kill it time.sleep(4.0) utils.run(['pkill', 'vtaction']) out, err = sp.communicate() # check the vtctl command got the right remote error back if "vtaction interrupted by signal" not in err: self.fail("cannot find expected output in error: " + err) logging.debug("vtaction was interrupted correctly:\n" + err) tablet_62344.kill_vttablet()
def run_vtworker(clargs, log_level='', auto_log=False, expect_fail=False, **kwargs): args = [ environment.binary_path('vtworker'), '-log_dir', environment.vtlogroot, '-port', str(environment.reserve_ports(1)) ] if auto_log: if options.verbose == 2: log_level = 'INFO' elif options.verbose == 1: log_level = 'WARNING' else: log_level = 'ERROR' if log_level: args.append('--stderrthreshold=%s' % log_level) cmd = args + clargs if expect_fail: return run_fail(cmd, **kwargs) return run(cmd, **kwargs)
def test_vtgate_qps(self): # create the topology utils.run_vtctl('CreateKeyspace test_keyspace') t = tablet.Tablet(tablet_uid=1, cell="nj") t.init_tablet("master", "test_keyspace", "0") t.update_addrs() utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True) # start vtgate and the qps-er vtgate_proc, vtgate_port = utils.vtgate_start() qpser = utils.run_bg(environment.binary_path('zkclient2')+' -server localhost:%u -mode qps2 test_nj test_keyspace' % vtgate_port) time.sleep(10) utils.kill_sub_process(qpser) # get the vtgate vars, make sure we have what we need v = utils.get_vars(vtgate_port) # some checks on performance / stats # a typical workstation will do 38-40k QPS, check we have more than 10k rpcCalls = v['TopoReaderRpcQueryCount']['test_nj'] if rpcCalls < 100000: self.fail('QPS is too low: %u < 10000' % (rpcCalls / 10)) else: logging.debug("Recorded qps: %u", rpcCalls / 10) utils.vtgate_kill(vtgate_proc)
def run_vtctl(clargs, log_level='', auto_log=False, expect_fail=False, **kwargs): args = [ environment.binary_path('vtctl'), '-log_dir', environment.vtlogroot ] args.extend(environment.topo_server_flags()) args.extend(environment.tablet_manager_protocol_flags()) args.extend(environment.tabletconn_protocol_flags()) if auto_log: if options.verbose == 2: log_level = 'INFO' elif options.verbose == 1: log_level = 'WARNING' else: log_level = 'ERROR' if log_level: args.append('--stderrthreshold=%s' % log_level) if isinstance(clargs, str): cmd = " ".join(args) + ' ' + clargs else: cmd = args + clargs if expect_fail: return run_fail(cmd, **kwargs) return run(cmd, **kwargs)
def test_zkocc_qps(self): # preload the test_nj cell zkocc_14850 = utils.zkocc_start() qpser = utils.run_bg(environment.binary_path('zkclient2')+' -server localhost:%u -mode qps /zk/test_nj/vt/zkocc1/data1 /zk/test_nj/vt/zkocc1/data2' % environment.zkocc_port_base) time.sleep(10) utils.kill_sub_process(qpser) # get the zkocc vars, make sure we have what we need v = utils.get_vars(environment.zkocc_port_base) if v['ZkReader']['test_nj']['State'] != 'Connected': self.fail('invalid zk global state: ' + v['ZkReader']['test_nj']['State']) # some checks on performance / stats # a typical workstation will do 45-47k QPS, check we have more than 10k rpcCalls = v['ZkReader']['RpcCalls'] if rpcCalls < 100000: self.fail('QPS is too low: %u < 10000' % (rpcCalls / 10)) else: logging.debug("Recorded qps: %u", rpcCalls / 10) cacheReads = v['ZkReader']['test_nj']['CacheReads'] if cacheReads < 100000: self.fail('Cache QPS is too low: %u < 10000' % (cacheReads / 10)) totalCacheReads = v['ZkReader']['total']['CacheReads'] self.assertEqual(cacheReads, totalCacheReads, 'Rollup stats are wrong') self.assertEqual(v['ZkReader']['UnknownCellErrors'], 0, 'unexpected UnknownCellErrors') utils.zkocc_kill(zkocc_14850)
def run_vtctl(clargs, log_level='', auto_log=False, expect_fail=False, **kwargs): args = [environment.binary_path('vtctl'), '-log_dir', environment.vtlogroot] args.extend(environment.topo_server_flags()) args.extend(environment.tablet_manager_protocol_flags()) args.extend(environment.tabletconn_protocol_flags()) if auto_log: if options.verbose == 2: log_level='INFO' elif options.verbose == 1: log_level='WARNING' else: log_level='ERROR' if log_level: args.append('--stderrthreshold=%s' % log_level) if isinstance(clargs, str): cmd = " ".join(args) + ' ' + clargs else: cmd = args + clargs if expect_fail: return run_fail(cmd, **kwargs) return run(cmd, **kwargs)
def vtgate_start(vtport=None, cell='test_nj', retry_delay=1, retry_count=1, topo_impl=None, tablet_bson_encrypted=False, cache_ttl='1s', auth=False, timeout="5s", cert=None, key=None, ca_cert=None): port = vtport or environment.reserve_ports(1) secure_port = None args = [environment.binary_path('vtgate'), '-port', str(port), '-cell', cell, '-retry-delay', '%ss' % (str(retry_delay)), '-retry-count', str(retry_count), '-log_dir', environment.vtlogroot, '-srv_topo_cache_ttl', cache_ttl, '-timeout', timeout, ] + environment.tabletconn_protocol_flags() if topo_impl: args.extend(['-topo_implementation', topo_impl]) else: args.extend(environment.topo_server_flags()) if tablet_bson_encrypted: args.append('-tablet-bson-encrypted') if auth: args.extend(['-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json')]) if cert: secure_port = environment.reserve_ports(1) args.extend(['-secure-port', '%s' % secure_port, '-cert', cert, '-key', key]) if ca_cert: args.extend(['-ca-cert', ca_cert]) sp = run_bg(args) if cert: wait_for_vars("vtgate", port, "SecureConnections") return sp, port, secure_port else: wait_for_vars("vtgate", port) return sp, port
def test_sigterm(self): utils.run_vtctl('CreateKeyspace test_keyspace') # create the database so vttablets start, as it is serving tablet_62344.create_db('vt_test_keyspace') tablet_62344.init_tablet('master', 'test_keyspace', '0', start=True) # start a 'vtctl Sleep' command in the background args = [environment.binary_path('vtctl'), '-log_dir', environment.vtlogroot, '--alsologtostderr'] args.extend(environment.topo_server_flags()) args.extend(environment.tablet_manager_protocol_flags()) args.extend(['Sleep', tablet_62344.tablet_alias, '60s']) sp = utils.run_bg(args, stdout=PIPE, stderr=PIPE) # wait for it to start, and let's kill it time.sleep(4.0) utils.run(['pkill', 'vtaction']) out, err = sp.communicate() # check the vtctl command got the right remote error back if "vtaction interrupted by signal" not in err: self.fail("cannot find expected output in error: " + err) logging.debug("vtaction was interrupted correctly:\n" + err) tablet_62344.kill_vttablet()
def test_get_srv_keyspace(self): utils.run_vtctl('CreateKeyspace test_keyspace') t = tablet.Tablet(tablet_uid=1, cell="nj") t.init_tablet("master", "test_keyspace", "0") utils.run_vtctl('UpdateTabletAddrs -hostname localhost -ip-addr 127.0.0.1 -mysql-port %s -vts-port %s %s' % (t.mysql_port, t.port + 500, t.tablet_alias)) self.rebuild() # vtgate zk API test out, err = utils.run(environment.binary_path('zkclient2')+' -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace' % self.vtgate_zk_port, trap_output=True) self.assertEqual(err, "Partitions[master] =\n" + " Shards[0]={Start: , End: }\n" + "Partitions[rdonly] =\n" + " Shards[0]={Start: , End: }\n" + "Partitions[replica] =\n" + " Shards[0]={Start: , End: }\n" + "Shards[0]={Start: , End: }\n" + "TabletTypes[0] = master\n", "Got wrong content: %s" % err) if environment.topo_server_implementation == 'zookeeper': reply = self.topo.get_srv_keyspace("test_nj", "test_keyspace") self.assertEqual(reply['TabletTypes'], ['master']) # zkocc API test out, err = utils.run(environment.binary_path('zkclient2')+' -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace' % environment.zkocc_port_base, trap_output=True) self.assertEqual(err, "Partitions[master] =\n" + " Shards[0]={Start: , End: }\n" + "Partitions[rdonly] =\n" + " Shards[0]={Start: , End: }\n" + "Partitions[replica] =\n" + " Shards[0]={Start: , End: }\n" + "Shards[0]={Start: , End: }\n" + "TabletTypes[0] = master\n", "Got wrong content: %s" % err) # vtgate zkocc API test out, err = utils.run(environment.binary_path('zkclient2')+' -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace' % self.vtgate_zkocc_port, trap_output=True) self.assertEqual(err, "Partitions[master] =\n" + " Shards[0]={Start: , End: }\n" + "Partitions[rdonly] =\n" + " Shards[0]={Start: , End: }\n" + "Partitions[replica] =\n" + " Shards[0]={Start: , End: }\n" + "Shards[0]={Start: , End: }\n" + "TabletTypes[0] = master\n", "Got wrong content: %s" % err)
def test_primecache(self): utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) master.init_tablet( 'master', 'test_keyspace', '0') replica.init_tablet('idle') utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True) master.create_db('vt_test_keyspace') master.start_vttablet(wait_for_state=None) replica.start_vttablet(wait_for_state=None) master.wait_for_vttablet_state('SERVING') replica.wait_for_vttablet_state('NOT_SERVING') # DB doesn't exist self._create_data() # we use clone to not prime the mysql cache on the slave db utils.run_vtctl(['Clone', '-force', '-server-mode', master.tablet_alias, replica.tablet_alias], auto_log=True) # sync the buffer cache, and clear it. This will prompt for user's password utils.run(['sync']) utils.run(['sudo', 'bash', '-c', 'echo 1 > /proc/sys/vm/drop_caches']) # we can now change data on the master for 30s, while slave is stopped. # master's binlog will be in OS buffer cache now. replica.mquery('', 'slave stop') self._change_random_data() use_primecache = True # easy to test without if use_primecache: # starting vtprimecache, sleeping for a couple seconds args = [environment.binary_path('vtprimecache'), '-db-config-dba-uname', 'vt_dba', '-db-config-dba-charset', 'utf8', '-db-config-dba-dbname', 'vt_test_keyspace', '-db-config-app-uname', 'vt_app', '-db-config-app-charset', 'utf8', '-db-config-app-dbname', 'vt_test_keyspace', '-relay_logs_path', replica.tablet_dir+'/relay-logs', '-mysql_socket_file', replica.tablet_dir+'/mysql.sock', '-log_dir', environment.vtlogroot, '-worker_count', '4', '-alsologtostderr', ] vtprimecache = utils.run_bg(args) time.sleep(2) # start slave, see how longs it takes to catch up on replication replica.mquery('', 'slave start') self.catch_up() if use_primecache: # TODO(alainjobart): read and check stats utils.kill_sub_process(vtprimecache) tablet.kill_tablets([master, replica])
def zkocc_start(cells=['test_nj'], extra_params=[]): args = [environment.binary_path('zkocc'), '-port', str(environment.zkocc_port_base), '-stderrthreshold=ERROR', ] + extra_params + cells sp = run_bg(args) wait_for_vars("zkocc", environment.zkocc_port_base) return sp
def vtgate_start(vtport=None, cell='test_nj', retry_delay=1, retry_count=1, topo_impl=None, tablet_bson_encrypted=False, cache_ttl='1s', auth=False, timeout="5s", cert=None, key=None, ca_cert=None): port = vtport or environment.reserve_ports(1) secure_port = None args = [ environment.binary_path('vtgate'), '-port', str(port), '-cell', cell, '-retry-delay', '%ss' % (str(retry_delay)), '-retry-count', str(retry_count), '-log_dir', environment.vtlogroot, '-srv_topo_cache_ttl', cache_ttl, '-timeout', timeout, ] + environment.tabletconn_protocol_flags() if topo_impl: args.extend(['-topo_implementation', topo_impl]) else: args.extend(environment.topo_server_flags()) if tablet_bson_encrypted: args.append('-tablet-bson-encrypted') if auth: args.extend([ '-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json') ]) if cert: secure_port = environment.reserve_ports(1) args.extend( ['-secure-port', '%s' % secure_port, '-cert', cert, '-key', key]) if ca_cert: args.extend(['-ca-cert', ca_cert]) sp = run_bg(args) if cert: wait_for_vars("vtgate", port, "SecureConnections") return sp, port, secure_port else: wait_for_vars("vtgate", port) return sp, port
def zk_teardown(): global zk_port_base zk_ports = ":".join( [str(zk_port_base), str(zk_port_base + 1), str(zk_port_base + 2)]) run('%s -log_dir %s -zk.cfg 1@%s:%s teardown' % (environment.binary_path('zkctl'), environment.vtlogroot, hostname, zk_ports), raise_on_error=False)
def test_get_end_points(self): utils.run_vtctl('CreateKeyspace test_keyspace') t = tablet.Tablet(tablet_uid=1, cell="nj") t.init_tablet("master", "test_keyspace", "0") t.update_addrs() self.rebuild() # vtgate zk API test out, err = utils.run(environment.binary_path('zkclient2')+' -server localhost:%u -mode getEndPoints test_nj test_keyspace 0 master' % self.vtgate_zk_port, trap_output=True) self.assertEqual(err, "Entries[0] = 1 localhost\n") if environment.topo_server_implementation == 'zookeeper': self.assertEqual(len(self.topo.get_end_points("test_nj", "test_keyspace", "0", "master")['Entries']), 1) # zkocc API test out, err = utils.run(environment.binary_path('zkclient2')+' -server localhost:%u -mode getEndPoints test_nj test_keyspace 0 master' % environment.zkocc_port_base, trap_output=True) self.assertEqual(err, "Entries[0] = 1 localhost\n") # vtgate zkocc API test out, err = utils.run(environment.binary_path('zkclient2')+' -server localhost:%u -mode getEndPoints test_nj test_keyspace 0 master' % self.vtgate_zkocc_port, trap_output=True) self.assertEqual(err, "Entries[0] = 1 localhost\n")
def start(self): args = [environment.binary_path('vtctld'), '-debug', '-templates', environment.vttop + '/go/cmd/vtctld/templates', '-log_dir', environment.vtlogroot, '-port', str(self.port), ] + \ environment.topo_server_flags() + \ environment.tablet_manager_protocol_flags() stderr_fd = open(os.path.join(environment.tmproot, "vtctld.stderr"), "w") self.proc = utils.run_bg(args, stderr=stderr_fd) return self.proc
def zk_setup(add_bad_host=False): global zk_port_base zk_ports = ":".join( [str(zk_port_base), str(zk_port_base + 1), str(zk_port_base + 2)]) run('%s -log_dir %s -zk.cfg 1@%s:%s init' % (environment.binary_path('zkctl'), environment.vtlogroot, hostname, zk_ports)) config = environment.tmproot + '/test-zk-client-conf.json' with open(config, 'w') as f: ca_server = 'localhost:%u' % (zk_port_base + 2) if add_bad_host: ca_server += ',does.not.exists:1234' zk_cell_mapping = { 'test_nj': 'localhost:%u' % (zk_port_base + 2), 'test_ny': 'localhost:%u' % (zk_port_base + 2), 'test_ca': ca_server, 'global': 'localhost:%u' % (zk_port_base + 2), 'test_nj:_zkocc': 'localhost:%u,localhost:%u,localhost:%u' % (environment.zkocc_port_base, environment.zkocc_port_base + 1, environment.zkocc_port_base + 2), 'test_ny:_zkocc': 'localhost:%u' % (environment.zkocc_port_base), 'test_ca:_zkocc': 'localhost:%u' % (environment.zkocc_port_base), 'global:_zkocc': 'localhost:%u' % (environment.zkocc_port_base), } json.dump(zk_cell_mapping, f) os.putenv('ZK_CLIENT_CONFIG', config) run(environment.binary_path('zk') + ' touch -p /zk/test_nj/vt') run(environment.binary_path('zk') + ' touch -p /zk/test_ny/vt') run(environment.binary_path('zk') + ' touch -p /zk/test_ca/vt')
def setUp(self): environment.topo_server_wipe() if environment.topo_server_implementation == 'zookeeper': utils.run(environment.binary_path('zk')+' touch -p /zk/test_nj/vt/zkocc1') utils.run(environment.binary_path('zk')+' touch -p /zk/test_nj/vt/zkocc2') fd = tempfile.NamedTemporaryFile(dir=environment.tmproot, delete=False) filename1 = fd.name fd.write("Test data 1") fd.close() utils.run(environment.binary_path('zk')+' cp '+filename1+' /zk/test_nj/vt/zkocc1/data1') fd = tempfile.NamedTemporaryFile(dir=environment.tmproot, delete=False) filename2 = fd.name fd.write("Test data 2") fd.close() utils.run(environment.binary_path('zk')+' cp '+filename2+' /zk/test_nj/vt/zkocc1/data2') fd = tempfile.NamedTemporaryFile(dir=environment.tmproot, delete=False) filename3 = fd.name fd.write("Test data 3") fd.close() utils.run(environment.binary_path('zk')+' cp '+filename3+' /zk/test_nj/vt/zkocc1/data3')
def zk_setup(add_bad_host=False): global zk_port_base zk_ports = ":".join([str(zk_port_base), str(zk_port_base+1), str(zk_port_base+2)]) run('%s -log_dir %s -zk.cfg 1@%s:%s init' % (environment.binary_path('zkctl'), environment.vtlogroot, hostname, zk_ports)) config = environment.tmproot+'/test-zk-client-conf.json' with open(config, 'w') as f: ca_server = 'localhost:%u' % (zk_port_base+2) if add_bad_host: ca_server += ',does.not.exists:1234' zk_cell_mapping = {'test_nj': 'localhost:%u'%(zk_port_base+2), 'test_ny': 'localhost:%u'%(zk_port_base+2), 'test_ca': ca_server, 'global': 'localhost:%u'%(zk_port_base+2), 'test_nj:_zkocc': 'localhost:%u,localhost:%u,localhost:%u'%(environment.zkocc_port_base,environment.zkocc_port_base+1,environment.zkocc_port_base+2), 'test_ny:_zkocc': 'localhost:%u'%(environment.zkocc_port_base), 'test_ca:_zkocc': 'localhost:%u'%(environment.zkocc_port_base), 'global:_zkocc': 'localhost:%u'%(environment.zkocc_port_base),} json.dump(zk_cell_mapping, f) os.putenv('ZK_CLIENT_CONFIG', config) run(environment.binary_path('zk')+' touch -p /zk/test_nj/vt') run(environment.binary_path('zk')+' touch -p /zk/test_ny/vt') run(environment.binary_path('zk')+' touch -p /zk/test_ca/vt')
def mysqlctl(self, cmd, quiet=False, extra_my_cnf=None, with_ports=False): env = None if extra_my_cnf: env = os.environ.copy() env['EXTRA_MY_CNF'] = extra_my_cnf ports_params = [] if with_ports: ports_params = ['-port', str(self.port), '-mysql-port', str(self.mysql_port)] return utils.run_bg([environment.binary_path('mysqlctl'), '-log_dir', environment.vtlogroot, '-tablet-uid', str(self.tablet_uid)] + ports_params + cmd, env=env)
def zk_wipe(): # Work around safety check on recursive delete. run(environment.binary_path('zk')+' rm -rf /zk/test_nj/vt/*') run(environment.binary_path('zk')+' rm -rf /zk/test_ny/vt/*') run(environment.binary_path('zk')+' rm -rf /zk/global/vt/*') run(environment.binary_path('zk')+' rm -f /zk/test_nj/vt') run(environment.binary_path('zk')+' rm -f /zk/test_ny/vt') run(environment.binary_path('zk')+' rm -f /zk/global/vt')
def zk_wipe(): # Work around safety check on recursive delete. run(environment.binary_path('zk') + ' rm -rf /zk/test_nj/vt/*') run(environment.binary_path('zk') + ' rm -rf /zk/test_ny/vt/*') run(environment.binary_path('zk') + ' rm -rf /zk/global/vt/*') run(environment.binary_path('zk') + ' rm -f /zk/test_nj/vt') run(environment.binary_path('zk') + ' rm -f /zk/test_ny/vt') run(environment.binary_path('zk') + ' rm -f /zk/global/vt')
def vtgate_start(cell='test_nj', retry_delay=1, retry_count=1, topo_impl=None, tablet_bson_encrypted=False): port = environment.reserve_ports(1) args = [environment.binary_path('vtgate'), '-port', str(port), '-cell', cell, '-retry-delay', '%ss' % (str(retry_delay)), '-retry-count', str(retry_count), '-log_dir', environment.vtlogroot, ] + environment.vtgate_protocol_flags() if topo_impl: args.extend(['-topo_implementation', topo_impl]) else: args.extend(environment.topo_server_flags()) if tablet_bson_encrypted: args.append('-tablet-bson-encrypted') sp = run_bg(args) wait_for_vars("vtgate", port) return sp, port
def run_vtworker(clargs, log_level='', auto_log=False, expect_fail=False, **kwargs): args = [environment.binary_path('vtworker'), '-log_dir', environment.vtlogroot, '-port', str(environment.reserve_ports(1))] if auto_log: if options.verbose == 2: log_level='INFO' elif options.verbose == 1: log_level='WARNING' else: log_level='ERROR' if log_level: args.append('--stderrthreshold=%s' % log_level) cmd = args + clargs if expect_fail: return run_fail(cmd, **kwargs) return run(cmd, **kwargs)
def mysqlctl(self, cmd, extra_my_cnf=None, with_ports=False): all_extra_my_cnf = [] if environment.mysql_flavor == "GoogleMysql": # we have to manually enable hierarchical replication to support groupid all_extra_my_cnf.append(environment.vttop + "/config/mycnf/master_google.cnf") if extra_my_cnf: all_extra_my_cnf.append(extra_my_cnf) env = None if all_extra_my_cnf: env = os.environ.copy() env['EXTRA_MY_CNF'] = ":".join(all_extra_my_cnf) ports_params = [] if with_ports: ports_params = ['-port', str(self.port), '-mysql-port', str(self.mysql_port)] return utils.run_bg([environment.binary_path('mysqlctl'), '-log_dir', environment.vtlogroot, '-tablet-uid', str(self.tablet_uid)] + ports_params + cmd, env=env)
def mysqlctl(self, cmd, extra_my_cnf=None, with_ports=False): all_extra_my_cnf = [] if environment.mysql_flavor == "GoogleMysql": # we have to manually enable hierarchical replication to support groupid all_extra_my_cnf.append(environment.vttop + "/config/mycnf/master_google.cnf") if extra_my_cnf: all_extra_my_cnf.append(extra_my_cnf) env = None if all_extra_my_cnf: env = os.environ.copy() env['EXTRA_MY_CNF'] = ":".join(all_extra_my_cnf) args = [environment.binary_path('mysqlctl'), '-log_dir', environment.vtlogroot, '-tablet_uid', str(self.tablet_uid)] if with_ports: args.extend(['-port', str(self.port), '-mysql_port', str(self.mysql_port)]) if utils.options.verbose == 2: args.append('-alsologtostderr') args.extend(cmd) return utils.run_bg(args, env=env)
def mysqlctl(self, cmd, quiet=False, extra_my_cnf=None, with_ports=False): env = None if extra_my_cnf: env = os.environ.copy() env["EXTRA_MY_CNF"] = extra_my_cnf ports_params = [] if with_ports: ports_params = ["-port", str(self.port), "-mysql-port", str(self.mysql_port)] return utils.run_bg( [ environment.binary_path("mysqlctl"), "-log_dir", environment.vtlogroot, "-tablet-uid", str(self.tablet_uid), ] + ports_params + cmd, env=env, )
def vtclient2(uid, path, query, bindvars=None, user=None, password=None, driver=None, verbose=False, raise_on_error=True): if (user is None) != (password is None): raise TypeError("you should provide either both or none of user and password") # for ZK paths to not have // in the path, that confuses things if path.startswith('/'): path = path[1:] server = "localhost:%u/%s" % (uid, path) if user is not None: server = "%s:%s@%s" % (user, password, server) cmdline = [environment.binary_path('vtclient2'), '-server', server] if bindvars: cmdline.extend(['-bindvars', bindvars]) if driver: cmdline.extend(['-driver', driver]) if verbose: cmdline.append('-verbose') cmdline.append(query) return run(cmdline, raise_on_error=raise_on_error, trap_output=True)
def zk_ls(path): out, err = run(environment.binary_path('zk') + ' ls ' + path, trap_output=True) return sorted(out.splitlines())
def zk_cat(path): out, err = run(environment.binary_path('zk')+' cat '+path, trap_output=True) return out
def zk_ls(path): out, err = run(environment.binary_path('zk')+' ls '+path, trap_output=True) return sorted(out.splitlines())
def zk_teardown(): global zk_port_base zk_ports = ":".join([str(zk_port_base), str(zk_port_base+1), str(zk_port_base+2)]) run('%s -log_dir %s -zk.cfg 1@%s:%s teardown' % (environment.binary_path('zkctl'), environment.vtlogroot, hostname, zk_ports), raise_on_error=False)
def zk_cat(path): out, err = run(environment.binary_path('zk') + ' cat ' + path, trap_output=True) return out
def test_primecache(self): utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) master.init_tablet('master', 'test_keyspace', '0') replica.init_tablet('idle') utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True) master.create_db('vt_test_keyspace') master.start_vttablet(wait_for_state=None) replica.start_vttablet(wait_for_state=None) master.wait_for_vttablet_state('SERVING') replica.wait_for_vttablet_state('NOT_SERVING') # DB doesn't exist self._create_data() # we use clone to not prime the mysql cache on the slave db utils.run_vtctl([ 'Clone', '-force', '-server-mode', master.tablet_alias, replica.tablet_alias ], auto_log=True) # sync the buffer cache, and clear it. This will prompt for user's password utils.run(['sync']) utils.run(['sudo', 'bash', '-c', 'echo 1 > /proc/sys/vm/drop_caches']) # we can now change data on the master for 30s, while slave is stopped. # master's binlog will be in OS buffer cache now. replica.mquery('', 'slave stop') self._change_random_data() use_primecache = True # easy to test without if use_primecache: # starting vtprimecache, sleeping for a couple seconds args = [ environment.binary_path('vtprimecache'), '-db-config-dba-uname', 'vt_dba', '-db-config-dba-charset', 'utf8', '-db-config-dba-dbname', 'vt_test_keyspace', '-db-config-app-uname', 'vt_app', '-db-config-app-charset', 'utf8', '-db-config-app-dbname', 'vt_test_keyspace', '-relay_logs_path', replica.tablet_dir + '/relay-logs', '-mysql_socket_file', replica.tablet_dir + '/mysql.sock', '-log_dir', environment.vtlogroot, '-worker_count', '4', '-alsologtostderr', ] vtprimecache = utils.run_bg(args) time.sleep(2) # start slave, see how longs it takes to catch up on replication replica.mquery('', 'slave start') self.catch_up() if use_primecache: # TODO(alainjobart): read and check stats utils.kill_sub_process(vtprimecache) tablet.kill_tablets([master, replica])
def test_reparent_down_master(self): utils.run_vtctl('CreateKeyspace test_keyspace') # create the database so vttablets start, as they are serving tablet_62344.create_db('vt_test_keyspace') tablet_62044.create_db('vt_test_keyspace') tablet_41983.create_db('vt_test_keyspace') tablet_31981.create_db('vt_test_keyspace') # Start up a master mysql and vttablet tablet_62344.init_tablet('master', 'test_keyspace', '0', start=True, wait_for_start=False) # Create a few slaves for testing reparenting. tablet_62044.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False) tablet_41983.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False) tablet_31981.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False) # wait for all tablets to start for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]: t.wait_for_vttablet_state("SERVING") # Recompute the shard layout node - until you do that, it might not be valid. utils.run_vtctl('RebuildShardGraph test_keyspace/0') utils.validate_topology() # Force the slaves to reparent assuming that all the datasets are identical. for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]: t.reset_replication() utils.run_vtctl('ReparentShard -force test_keyspace/0 ' + tablet_62344.tablet_alias, auto_log=True) utils.validate_topology() # Make the master agent and database unavailable. tablet_62344.kill_vttablet() tablet_62344.shutdown_mysql().wait() self._check_db_addr('0', 'master', tablet_62344.port) # Perform a reparent operation - the Validate part will try to ping # the master and fail somewhat quickly stdout, stderr = utils.run_vtctl('-wait-time 5s ReparentShard test_keyspace/0 ' + tablet_62044.tablet_alias, expect_fail=True) logging.debug("Failed ReparentShard output:\n" + stderr) if 'ValidateShard verification failed' not in stderr: self.fail("didn't find the right error strings in failed ReparentShard: " + stderr) # Should timeout and fail stdout, stderr = utils.run_vtctl('-wait-time 5s ScrapTablet ' + tablet_62344.tablet_alias, expect_fail=True) logging.debug("Failed ScrapTablet output:\n" + stderr) if 'deadline exceeded' not in stderr: self.fail("didn't find the right error strings in failed ScrapTablet: " + stderr) # Should interrupt and fail args = [environment.binary_path('vtctl'), '-log_dir', environment.vtlogroot, '-wait-time', '10s'] args.extend(environment.topo_server_flags()) args.extend(environment.tablet_manager_protocol_flags()) args.extend(['ScrapTablet', tablet_62344.tablet_alias]) sp = utils.run_bg(args, stdout=PIPE, stderr=PIPE) # Need time for the process to start before killing it. time.sleep(3.0) os.kill(sp.pid, signal.SIGINT) stdout, stderr = sp.communicate() logging.debug("Failed ScrapTablet output:\n" + stderr) if 'interrupted' not in stderr: self.fail("didn't find the right error strings in failed ScrapTablet: " + stderr) # Force the scrap action in zk even though tablet is not accessible. tablet_62344.scrap(force=True) utils.run_vtctl('ChangeSlaveType -force %s idle' % tablet_62344.tablet_alias, expect_fail=True) # Remove pending locks (make this the force option to ReparentShard?) if environment.topo_server_implementation == 'zookeeper': utils.run_vtctl('PurgeActions /zk/global/vt/keyspaces/test_keyspace/shards/0/action') # Re-run reparent operation, this shoud now proceed unimpeded. utils.run_vtctl('-wait-time 1m ReparentShard test_keyspace/0 ' + tablet_62044.tablet_alias, auto_log=True) utils.validate_topology() self._check_db_addr('0', 'master', tablet_62044.port) utils.run_vtctl(['ChangeSlaveType', '-force', tablet_62344.tablet_alias, 'idle']) idle_tablets, _ = utils.run_vtctl('ListAllTablets test_nj', trap_output=True) if '0000062344 <null> <null> idle' not in idle_tablets: self.fail('idle tablet not found: %s' % idle_tablets) tablet.kill_tablets([tablet_62044, tablet_41983, tablet_31981]) # so the other tests don't have any surprise tablet_62344.start_mysql().wait()
def _test_reparent_from_outside(self, brutal=False): utils.run_vtctl('CreateKeyspace test_keyspace') # create the database so vttablets start, as they are serving for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]: t.create_db('vt_test_keyspace') # Start up a master mysql and vttablet tablet_62344.init_tablet('master', 'test_keyspace', '0', start=True, wait_for_start=False) # Create a few slaves for testing reparenting. tablet_62044.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False) tablet_41983.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False) tablet_31981.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False) # wait for all tablets to start for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]: t.wait_for_vttablet_state("SERVING") # Reparent as a starting point for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]: t.reset_replication() utils.run_vtctl('ReparentShard -force test_keyspace/0 %s' % tablet_62344.tablet_alias, auto_log=True) # now manually reparent 1 out of 2 tablets # 62044 will be the new master # 31981 won't be re-parented, so it will be busted tablet_62044.mquery('', [ "RESET MASTER", "STOP SLAVE", "RESET SLAVE", "CHANGE MASTER TO MASTER_HOST = ''", ]) new_pos = tablet_62044.mquery('', 'show master status') logging.debug("New master position: %s" % str(new_pos)) # 62344 will now be a slave of 62044 tablet_62344.mquery('', [ "RESET MASTER", "RESET SLAVE", "change master to master_host='%s', master_port=%u, master_log_file='%s', master_log_pos=%u" % (utils.hostname, tablet_62044.mysql_port, new_pos[0][0], new_pos[0][1]), 'start slave' ]) # 41983 will be a slave of 62044 tablet_41983.mquery('', [ 'stop slave', "change master to master_port=%u, master_log_file='%s', master_log_pos=%u" % (tablet_62044.mysql_port, new_pos[0][0], new_pos[0][1]), 'start slave' ]) # in brutal mode, we scrap the old master first if brutal: tablet_62344.scrap(force=True) # we have some automated tools that do this too, so it's good to simulate if environment.topo_server_implementation == 'zookeeper': utils.run(environment.binary_path('zk')+' rm -rf ' + tablet_62344.zk_tablet_path) # update zk with the new graph utils.run_vtctl('ShardExternallyReparented test_keyspace/0 %s' % tablet_62044.tablet_alias, auto_log=True) self._test_reparent_from_outside_check(brutal) utils.run_vtctl('RebuildReplicationGraph test_nj test_keyspace') self._test_reparent_from_outside_check(brutal) tablet.kill_tablets([tablet_31981, tablet_62344, tablet_62044, tablet_41983])
def _test_reparent_from_outside(self, brutal=False): utils.run_vtctl('CreateKeyspace test_keyspace') # create the database so vttablets start, as they are serving for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]: t.create_db('vt_test_keyspace') # Start up a master mysql and vttablet tablet_62344.init_tablet('master', 'test_keyspace', '0', start=True, wait_for_start=False) # Create a few slaves for testing reparenting. tablet_62044.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False) tablet_41983.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False) tablet_31981.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False) # wait for all tablets to start for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]: t.wait_for_vttablet_state("SERVING") # Reparent as a starting point for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]: t.reset_replication() utils.run_vtctl('ReparentShard -force test_keyspace/0 %s' % tablet_62344.tablet_alias, auto_log=True) # now manually reparent 1 out of 2 tablets # 62044 will be the new master # 31981 won't be re-parented, so it will be busted tablet_62044.mquery('', [ "RESET MASTER", "STOP SLAVE", "RESET SLAVE", "CHANGE MASTER TO MASTER_HOST = ''", ]) new_pos = tablet_62044.mquery('', 'show master status') logging.debug("New master position: %s" % str(new_pos)) # 62344 will now be a slave of 62044 tablet_62344.mquery('', [ "RESET MASTER", "RESET SLAVE", "change master to master_host='%s', master_port=%u, master_log_file='%s', master_log_pos=%u" % (utils.hostname, tablet_62044.mysql_port, new_pos[0][0], new_pos[0][1]), 'start slave' ]) # 41983 will be a slave of 62044 tablet_41983.mquery('', [ 'stop slave', "change master to master_port=%u, master_log_file='%s', master_log_pos=%u" % (tablet_62044.mysql_port, new_pos[0][0], new_pos[0][1]), 'start slave' ]) # in brutal mode, we scrap the old master first if brutal: tablet_62344.scrap(force=True) # we have some automated tools that do this too, so it's good to simulate if environment.topo_server_implementation == 'zookeeper': utils.run( environment.binary_path('zk') + ' rm -rf ' + tablet_62344.zk_tablet_path) # update zk with the new graph utils.run_vtctl('ShardExternallyReparented test_keyspace/0 %s' % tablet_62044.tablet_alias, auto_log=True) self._test_reparent_from_outside_check(brutal) utils.run_vtctl('RebuildReplicationGraph test_nj test_keyspace') self._test_reparent_from_outside_check(brutal) tablet.kill_tablets( [tablet_31981, tablet_62344, tablet_62044, tablet_41983])
def test_reparent_down_master(self): utils.run_vtctl('CreateKeyspace test_keyspace') # create the database so vttablets start, as they are serving tablet_62344.create_db('vt_test_keyspace') tablet_62044.create_db('vt_test_keyspace') tablet_41983.create_db('vt_test_keyspace') tablet_31981.create_db('vt_test_keyspace') # Start up a master mysql and vttablet tablet_62344.init_tablet('master', 'test_keyspace', '0', start=True, wait_for_start=False) # Create a few slaves for testing reparenting. tablet_62044.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False) tablet_41983.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False) tablet_31981.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False) # wait for all tablets to start for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]: t.wait_for_vttablet_state("SERVING") # Recompute the shard layout node - until you do that, it might not be valid. utils.run_vtctl('RebuildShardGraph test_keyspace/0') utils.validate_topology() # Force the slaves to reparent assuming that all the datasets are identical. for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]: t.reset_replication() utils.run_vtctl('ReparentShard -force test_keyspace/0 ' + tablet_62344.tablet_alias, auto_log=True) utils.validate_topology() # Make the master agent and database unavailable. tablet_62344.kill_vttablet() tablet_62344.shutdown_mysql().wait() self._check_db_addr('0', 'master', tablet_62344.port) # Perform a reparent operation - the Validate part will try to ping # the master and fail somewhat quickly stdout, stderr = utils.run_vtctl( '-wait-time 5s ReparentShard test_keyspace/0 ' + tablet_62044.tablet_alias, expect_fail=True) logging.debug("Failed ReparentShard output:\n" + stderr) if 'ValidateShard verification failed' not in stderr: self.fail( "didn't find the right error strings in failed ReparentShard: " + stderr) # Should timeout and fail stdout, stderr = utils.run_vtctl('-wait-time 5s ScrapTablet ' + tablet_62344.tablet_alias, expect_fail=True) logging.debug("Failed ScrapTablet output:\n" + stderr) if 'deadline exceeded' not in stderr: self.fail( "didn't find the right error strings in failed ScrapTablet: " + stderr) # Should interrupt and fail args = [ environment.binary_path('vtctl'), '-log_dir', environment.vtlogroot, '-wait-time', '10s' ] args.extend(environment.topo_server_flags()) args.extend(environment.tablet_manager_protocol_flags()) args.extend(['ScrapTablet', tablet_62344.tablet_alias]) sp = utils.run_bg(args, stdout=PIPE, stderr=PIPE) # Need time for the process to start before killing it. time.sleep(3.0) os.kill(sp.pid, signal.SIGINT) stdout, stderr = sp.communicate() logging.debug("Failed ScrapTablet output:\n" + stderr) if 'interrupted' not in stderr: self.fail( "didn't find the right error strings in failed ScrapTablet: " + stderr) # Force the scrap action in zk even though tablet is not accessible. tablet_62344.scrap(force=True) utils.run_vtctl('ChangeSlaveType -force %s idle' % tablet_62344.tablet_alias, expect_fail=True) # Remove pending locks (make this the force option to ReparentShard?) if environment.topo_server_implementation == 'zookeeper': utils.run_vtctl( 'PurgeActions /zk/global/vt/keyspaces/test_keyspace/shards/0/action' ) # Re-run reparent operation, this shoud now proceed unimpeded. utils.run_vtctl('-wait-time 1m ReparentShard test_keyspace/0 ' + tablet_62044.tablet_alias, auto_log=True) utils.validate_topology() self._check_db_addr('0', 'master', tablet_62044.port) utils.run_vtctl( ['ChangeSlaveType', '-force', tablet_62344.tablet_alias, 'idle']) idle_tablets, _ = utils.run_vtctl('ListAllTablets test_nj', trap_output=True) if '0000062344 <null> <null> idle' not in idle_tablets: self.fail('idle tablet not found: %s' % idle_tablets) tablet.kill_tablets([tablet_62044, tablet_41983, tablet_31981]) # so the other tests don't have any surprise tablet_62344.start_mysql().wait()
def test_sharding(self): shard_0_master.init_tablet('master', 'test_keyspace', '-80') shard_0_replica.init_tablet('replica', 'test_keyspace', '-80') shard_1_master.init_tablet('master', 'test_keyspace', '80-') shard_1_replica.init_tablet('replica', 'test_keyspace', '80-') utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True) # run checks now before we start the tablets utils.validate_topology() # create databases, start the tablets, wait for them to start for t in [ shard_0_master, shard_0_replica, shard_1_master, shard_1_replica ]: t.create_db('vt_test_keyspace') t.start_vttablet(wait_for_state=None) for t in [ shard_0_master, shard_0_replica, shard_1_master, shard_1_replica ]: t.wait_for_vttablet_state('SERVING') # apply the schema on the first shard through vtctl, so all tablets # are the same (replication is not enabled yet, so allow_replication=false # is just there to be tested) utils.run_vtctl([ 'ApplySchema', '-stop-replication', '-sql=' + create_vt_select_test.replace("\n", ""), shard_0_master.tablet_alias ]) utils.run_vtctl([ 'ApplySchema', '-stop-replication', '-sql=' + create_vt_select_test.replace("\n", ""), shard_0_replica.tablet_alias ]) if environment.topo_server_implementation == 'zookeeper': # start zkocc, we'll use it later, indirectly with the vtdb-zkocc driver zkocc_server = utils.zkocc_start() # start vtgate, we'll use it later vtgate_server, vtgate_port = utils.vtgate_start() for t in [ shard_0_master, shard_0_replica, shard_1_master, shard_1_replica ]: t.reset_replication() utils.run_vtctl('ReparentShard -force test_keyspace/-80 ' + shard_0_master.tablet_alias, auto_log=True) utils.run_vtctl('ReparentShard -force test_keyspace/80- ' + shard_1_master.tablet_alias, auto_log=True) # apply the schema on the second shard using a simple schema upgrade utils.run_vtctl([ 'ApplySchemaShard', '-simple', '-sql=' + create_vt_select_test_reverse.replace("\n", ""), 'test_keyspace/80-' ]) # insert some values directly (db is RO after minority reparent) # FIXME(alainjobart) these values don't match the shard map utils.run_vtctl('SetReadWrite ' + shard_0_master.tablet_alias) utils.run_vtctl('SetReadWrite ' + shard_1_master.tablet_alias) shard_0_master.mquery( 'vt_test_keyspace', "insert into vt_select_test (id, msg) values (1, 'test 1')", write=True) shard_1_master.mquery( 'vt_test_keyspace', "insert into vt_select_test (id, msg) values (10, 'test 10')", write=True) utils.validate_topology(ping_tablets=True) utils.pause("Before the sql scatter query") # note the order of the rows is not guaranteed, as the go routines # doing the work can go out of order self._check_rows(["Index\tid\tmsg", "1\ttest 1", "10\ttest 10"]) # write a value, re-read them all utils.vtclient2( 3803, "/test_nj/test_keyspace/master", "insert into vt_select_test (id, msg) values (:keyspace_id, 'test 2')", bindvars='{"keyspace_id": 2}', driver="vtdb", verbose=True) self._check_rows( ["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"]) # make sure the '2' value was written on first shard rows = shard_0_master.mquery( 'vt_test_keyspace', "select id, msg from vt_select_test order by id") self.assertEqual(rows, ( (1, 'test 1'), (2, 'test 2'), ), 'wrong mysql_query output: %s' % str(rows)) utils.pause("After db writes") # now use various topo servers and streaming or both for the same query self._check_rows( ["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"], driver="vtdb-streaming") if environment.topo_server_implementation == 'zookeeper': self._check_rows( ["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"], driver="vtdb-zk") self._check_rows( ["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"], driver="vtdb-zk-streaming") self._check_rows( ["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"], driver="vtdb-zkocc") self._check_rows( ["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"], driver="vtdb-zkocc-streaming") # make sure the schema checking works self._check_rows_schema_diff("vtdb") if environment.topo_server_implementation == 'zookeeper': self._check_rows_schema_diff("vtdb-zk") self._check_rows_schema_diff("vtdb-zkocc") # throw in some schema validation step # we created the schema differently, so it should show utils.run_vtctl('ValidateSchemaShard test_keyspace/-80') utils.run_vtctl('ValidateSchemaShard test_keyspace/80-') out, err = utils.run_vtctl('ValidateSchemaKeyspace test_keyspace', trap_output=True, raise_on_error=False) if "test_nj-0000062344 and test_nj-0000062346 disagree on schema for table vt_select_test:\nCREATE TABLE" not in err or \ "test_nj-0000062344 and test_nj-0000062347 disagree on schema for table vt_select_test:\nCREATE TABLE" not in err: self.fail('wrong ValidateSchemaKeyspace output: ' + err) # validate versions utils.run_vtctl('ValidateVersionShard test_keyspace/-80', auto_log=True) utils.run_vtctl('ValidateVersionKeyspace test_keyspace', auto_log=True) # show and validate permissions utils.run_vtctl('GetPermissions test_nj-0000062344', auto_log=True) utils.run_vtctl('ValidatePermissionsShard test_keyspace/-80', auto_log=True) utils.run_vtctl('ValidatePermissionsKeyspace test_keyspace', auto_log=True) if environment.topo_server_implementation == 'zookeeper': # and create zkns on this complex keyspace, make sure a few files are created utils.run_vtctl('ExportZknsForKeyspace test_keyspace') out, err = utils.run(environment.binary_path('zk') + ' ls -R /zk/test_nj/zk?s/vt/test_keysp*', trap_output=True) lines = out.splitlines() for base in ['-80', '80-']: for db_type in ['master', 'replica']: for sub_path in ['', '.vdns', '/0', '/_vtocc.vdns']: expected = '/zk/test_nj/zkns/vt/test_keyspace/' + base + '/' + db_type + sub_path if expected not in lines: self.fail('missing zkns part:\n%s\nin:%s' % (expected, out)) # now try to connect using the python client and shard-aware connection # to both shards # first get the topology and check it vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port, "test_nj", 30.0) topology.read_keyspaces(vtgate_client) shard_0_master_addrs = topology.get_host_port_by_name( vtgate_client, "test_keyspace.-80.master:_vtocc") if len(shard_0_master_addrs) != 1: self.fail( 'topology.get_host_port_by_name failed for "test_keyspace.-80.master:_vtocc", got: %s' % " ".join([ "%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs ])) logging.debug( "shard 0 master addrs: %s", " ".join([ "%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs ])) # connect to shard -80 conn = tablet3.TabletConnection( "%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]), "", "test_keyspace", "-80", 10.0) conn.dial() (results, rowcount, lastrowid, fields) = conn._execute( "select id, msg from vt_select_test order by id", {}) self.assertEqual(results, [ (1, 'test 1'), (2, 'test 2'), ], 'wrong conn._execute output: %s' % str(results)) # connect to shard 80- shard_1_master_addrs = topology.get_host_port_by_name( vtgate_client, "test_keyspace.80-.master:_vtocc") conn = tablet3.TabletConnection( "%s:%u" % (shard_1_master_addrs[0][0], shard_1_master_addrs[0][1]), "", "test_keyspace", "80-", 10.0) conn.dial() (results, rowcount, lastrowid, fields) = conn._execute( "select id, msg from vt_select_test order by id", {}) self.assertEqual(results, [ (10, 'test 10'), ], 'wrong conn._execute output: %s' % str(results)) vtgate_client.close() # try to connect with bad shard try: conn = tablet3.TabletConnection( "localhost:%u" % shard_0_master.port, "", "test_keyspace", "-90", 10.0) conn.dial() self.fail('expected an exception') except Exception as e: if "fatal: Shard mismatch, expecting -80, received -90" not in str( e): self.fail('unexpected exception: ' + str(e)) utils.vtgate_kill(vtgate_server) if environment.topo_server_implementation == 'zookeeper': utils.kill_sub_process(zkocc_server) tablet.kill_tablets( [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica])
def test_zkocc(self): # preload the test_nj cell zkocc_14850 = utils.zkocc_start(extra_params=['-connect-timeout=2s', '-cache-refresh-interval=1s']) time.sleep(1) # create a python client. The first address is bad, will test the retry logic bad_port = environment.reserve_ports(3) zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u,localhost:%u" % (bad_port, environment.zkocc_port_base, bad_port+1), "test_nj", 30) zkocc_client.dial() # test failure for a python client that cannot connect bad_zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u" % (bad_port+2, bad_port), "test_nj", 30) try: bad_zkocc_client.dial() self.fail('exception expected') except zkocc.ZkOccError as e: if not str(e).startswith("Cannot dial to any server, tried: "): self.fail('unexpected exception: %s' % str(e)) level = logging.getLogger().getEffectiveLevel() logging.getLogger().setLevel(logging.ERROR) # FIXME(ryszard): This can be changed into a self.assertRaises. try: bad_zkocc_client.get("/zk/test_nj/vt/zkocc1/data1") self.fail('exception expected') except zkocc.ZkOccError as e: if not str(e).startswith("Cannot dial to any server, tried: "): self.fail('unexpected exception: %s' % str(e)) logging.getLogger().setLevel(level) # get test out, err = utils.run(environment.binary_path('zkclient2')+' -server localhost:%u /zk/test_nj/vt/zkocc1/data1' % environment.zkocc_port_base, trap_output=True) self.assertEqual(err, "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=false, Stale=false)\n") zk_data = zkocc_client.get("/zk/test_nj/vt/zkocc1/data1") self.assertDictContainsSubset({'Data': "Test data 1", 'Cached': True, 'Stale': False,}, zk_data) self.assertDictContainsSubset({'NumChildren': 0, 'Version': 0}, zk_data['Stat']) # getv test out, err = utils.run(environment.binary_path('zkclient2')+' -server localhost:%u /zk/test_nj/vt/zkocc1/data1 /zk/test_nj/vt/zkocc1/data2 /zk/test_nj/vt/zkocc1/data3' % environment.zkocc_port_base, trap_output=True) self.assertEqualNormalized(err, """[0] /zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false) [1] /zk/test_nj/vt/zkocc1/data2 = Test data 2 (NumChildren=0, Version=0, Cached=false, Stale=false) [2] /zk/test_nj/vt/zkocc1/data3 = Test data 3 (NumChildren=0, Version=0, Cached=false, Stale=false) """) zk_data = zkocc_client.getv(["/zk/test_nj/vt/zkocc1/data1", "/zk/test_nj/vt/zkocc1/data2", "/zk/test_nj/vt/zkocc1/data3"])['Nodes'] self.assertEqual(len(zk_data), 3) for i, d in enumerate(zk_data): self.assertEqual(d['Data'], 'Test data %s' % (i + 1)) self.assertTrue(d['Cached']) self.assertFalse(d['Stale']) self.assertDictContainsSubset({'NumChildren': 0, 'Version': 0}, d['Stat']) # children test out, err = utils.run(environment.binary_path('zkclient2')+' -server localhost:%u -mode children /zk/test_nj/vt' % environment.zkocc_port_base, trap_output=True) self.assertEqualNormalized(err, """Path = /zk/test_nj/vt Child[0] = zkocc1 Child[1] = zkocc2 NumChildren = 2 CVersion = 2 Cached = false Stale = false """) # zk command tests self._check_zk_output("cat /zk/test_nj/vt/zkocc1/data1", "Test data 1") self._check_zk_output("ls -l /zk/test_nj/vt/zkocc1", """total: 3 -rw-rw-rw- zk zk 11 %s data1 -rw-rw-rw- zk zk 11 %s data2 -rw-rw-rw- zk zk 11 %s data3 """ % (_format_time(zk_data[0]['Stat']['MTime']), _format_time(zk_data[1]['Stat']['MTime']), _format_time(zk_data[2]['Stat']['MTime']))) # test /zk/local is not resolved and rejected out, err = utils.run(environment.binary_path('zkclient2')+' -server localhost:%u /zk/local/vt/zkocc1/data1' % environment.zkocc_port_base, trap_output=True, raise_on_error=False) self.assertIn("zkocc: cannot resolve local cell", err) # start a background process to query the same value over and over again # while we kill the zk server and restart it outfd = tempfile.NamedTemporaryFile(dir=environment.tmproot, delete=False) filename = outfd.name querier = utils.run_bg('/bin/bash -c "while true ; do '+environment.binary_path('zkclient2')+' -server localhost:%u /zk/test_nj/vt/zkocc1/data1 ; sleep 0.1 ; done"' % environment.zkocc_port_base, stderr=outfd.file) outfd.close() time.sleep(1) # kill zk server, sleep a bit, restart zk server, sleep a bit utils.run(environment.binary_path('zkctl')+' -zk.cfg 1@'+utils.hostname+':%u:%u:%u shutdown' % (environment.zk_port_base, environment.zk_port_base+1, environment.zk_port_base+2)) time.sleep(3) utils.run(environment.binary_path('zkctl')+' -zk.cfg 1@'+utils.hostname+':%u:%u:%u start' % (environment.zk_port_base, environment.zk_port_base+1, environment.zk_port_base+2)) time.sleep(3) utils.kill_sub_process(querier) logging.debug("Checking %s", filename) fd = open(filename, "r") state = 0 for line in fd: if line == "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)\n": stale = False elif line == "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=true)\n": stale = True else: self.fail('unexpected line: %s' % line) if state == 0: if stale: state = 1 elif state == 1: if not stale: state = 2 else: if stale: self.fail('unexpected stale state') self.assertEqual(state, 2) fd.close() utils.zkocc_kill(zkocc_14850) # check that after the server is gone, the python client fails correctly level = logging.getLogger().getEffectiveLevel() logging.getLogger().setLevel(logging.ERROR) try: zkocc_client.get("/zk/test_nj/vt/zkocc1/data1") self.fail('exception expected') except zkocc.ZkOccError as e: if not str(e).startswith("Cannot dial to any server, tried: "): self.fail('unexpected exception: %s', str(e)) logging.getLogger().setLevel(level)
def start_vttablet(self, port=None, auth=False, memcache=False, wait_for_state="SERVING", customrules=None, schema_override=None, cert=None, key=None, ca_cert=None, repl_extra_flags={}, sensitive_mode=False, target_tablet_type=None, lameduck_period=None, extra_args=None, full_mycnf_args=False, security_policy=None): """ Starts a vttablet process, and returns it. The process is also saved in self.proc, so it's easy to kill as well. """ environment.prog_compile('vtaction') args = [ environment.binary_path('vttablet'), '-port', '%s' % (port or self.port), '-tablet-path', self.tablet_alias, '-log_dir', environment.vtlogroot ] args.extend(environment.topo_server_flags()) args.extend(utils.binlog_player_protocol_flags) dbconfigs = self._get_db_configs_file(repl_extra_flags) for key1 in dbconfigs: for key2 in dbconfigs[key1]: args.extend( ["-db-config-" + key1 + "-" + key2, dbconfigs[key1][key2]]) if full_mycnf_args: # this flag is used to specify all the mycnf_ flags, to make # sure that code works and can fork actions. relay_log_path = os.path.join( self.tablet_dir, "relay-logs", "vt-%010d-relay-bin" % self.tablet_uid) args.extend([ "-mycnf_server_id", str(self.tablet_uid), "-mycnf_mysql_port", str(self.mysql_port), "-mycnf_data_dir", os.path.join(self.tablet_dir, "data"), "-mycnf_innodb_data_home_dir", os.path.join(self.tablet_dir, "innodb", "data"), "-mycnf_innodb_log_group_home_dir", os.path.join(self.tablet_dir, "innodb", "logs"), "-mycnf_socket_file", os.path.join(self.tablet_dir, "mysql.sock"), "-mycnf_error_log_path", os.path.join(self.tablet_dir, "error.log"), "-mycnf_slow_log_path", os.path.join(self.tablet_dir, "slow-query.log"), "-mycnf_relay_log_path", relay_log_path, "-mycnf_relay_log_index_path", relay_log_path + ".index", "-mycnf_relay_log_info_path", os.path.join(self.tablet_dir, "relay-logs", "relay-log.info"), "-mycnf_bin_log_path", os.path.join(self.tablet_dir, "bin-logs", "vt-%010d-bin" % self.tablet_uid), "-mycnf_master_info_file", os.path.join(self.tablet_dir, "master.info"), "-mycnf_pid_file", os.path.join(self.tablet_dir, "mysql.pid"), "-mycnf_tmp_dir", os.path.join(self.tablet_dir, "tmp"), "-mycnf_slave_load_tmp_dir", os.path.join(self.tablet_dir, "tmp"), ]) if memcache: args.extend(["-rowcache-bin", environment.memcached_bin()]) memcache_socket = os.path.join(self.tablet_dir, "memcache.sock") args.extend(["-rowcache-socket", memcache_socket]) args.extend(["-enable-rowcache"]) if auth: args.extend([ '-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json') ]) if customrules: args.extend(['-customrules', customrules]) if schema_override: args.extend(['-schema-override', schema_override]) if cert: self.secure_port = environment.reserve_ports(1) args.extend([ '-secure-port', '%s' % self.secure_port, '-cert', cert, '-key', key ]) if ca_cert: args.extend(['-ca_cert', ca_cert]) if sensitive_mode: args.extend(['-queryserver-config-sensitive-mode']) if target_tablet_type: args.extend([ '-target_tablet_type', target_tablet_type, '-health_check_interval', '2s', '-allowed_replication_lag', '30' ]) if lameduck_period: args.extend(['-lameduck-period', lameduck_period]) if extra_args: args.extend(extra_args) if security_policy: args.extend(['-security_policy', security_policy]) stderr_fd = open(os.path.join(self.tablet_dir, "vttablet.stderr"), "w") # increment count only the first time if not self.proc: Tablet.tablets_running += 1 self.proc = utils.run_bg(args, stderr=stderr_fd) stderr_fd.close() # wait for zookeeper PID just to be sure we have it if environment.topo_server_implementation == 'zookeeper': utils.run(environment.binary_path('zk') + ' wait -e ' + self.zk_pid, stdout=utils.devnull) # wait for query service to be in the right state if wait_for_state: self.wait_for_vttablet_state(wait_for_state, port=port) return self.proc