def start_vttablet(self, port=None, auth=False, memcache=False, wait_for_state="SERVING", customrules=None, schema_override=None, cert=None, key=None, ca_cert=None, repl_extra_flags={}):
"""
Starts a vttablet process, and returns it.
The process is also saved in self.proc, so it's easy to kill as well.
"""
environment.prog_compile('vtaction')
args = [environment.binary_path('vttablet'),
'-port', '%s' % (port or self.port),
'-tablet-path', self.tablet_alias,
'-log_dir', environment.vtlogroot]
args.extend(environment.topo_server_flags())
args.extend(environment.binlog_player_protocol_flags())
dbconfigs = self._get_db_configs_file(repl_extra_flags)
for key1 in dbconfigs:
for key2 in dbconfigs[key1]:
args.extend(["-db-config-"+key1+"-"+key2, dbconfigs[key1][key2]])
if memcache:
if os.path.exists(environment.vtroot + "/bin/memcached"):
args.extend(["-rowcache-bin", environment.vtroot + "/bin/memcached"])
else:
args.extend(["-rowcache-bin", "memcached"])
memcache_socket = os.path.join(self.tablet_dir, "memcache.sock")
args.extend(["-rowcache-socket", memcache_socket])
args.extend(["-enable-rowcache"])
if auth:
args.extend(['-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json')])
if customrules:
args.extend(['-customrules', customrules])
if schema_override:
args.extend(['-schema-override', schema_override])
if cert:
self.secure_port = environment.reserve_ports(1)
args.extend(['-secure-port', '%s' % self.secure_port,
'-cert', cert,
'-key', key])
if ca_cert:
args.extend(['-ca-cert', ca_cert])
stderr_fd = open(os.path.join(self.tablet_dir, "vttablet.stderr"), "w")
# increment count only the first time
if not self.proc:
Tablet.tablets_running += 1
self.proc = utils.run_bg(args, stderr=stderr_fd)
stderr_fd.close()
# wait for zookeeper PID just to be sure we have it
if environment.topo_server_implementation == 'zookeeper':
utils.run(environment.binary_path('zk')+' wait -e ' + self.zk_pid, stdout=utils.devnull)
# wait for query service to be in the right state
if wait_for_state:
self.wait_for_vttablet_state(wait_for_state, port=port)
return self.proc
def start_vttablet(self, port=None, auth=False, memcache=False, wait_for_state="SERVING", customrules=None, schema_override=None, cert=None, key=None, ca_cert=None, repl_extra_flags={}):
"""
Starts a vttablet process, and returns it.
The process is also saved in self.proc, so it's easy to kill as well.
"""
environment.prog_compile('vtaction')
args = [environment.binary_path('vttablet'),
'-port', '%s' % (port or self.port),
'-tablet-path', self.tablet_alias,
'-log_dir', environment.vtlogroot]
args.extend(environment.topo_server_flags())
args.extend(utils.binlog_player_protocol_flags)
dbconfigs = self._get_db_configs_file(repl_extra_flags)
for key1 in dbconfigs:
for key2 in dbconfigs[key1]:
args.extend(["-db-config-"+key1+"-"+key2, dbconfigs[key1][key2]])
if memcache:
if os.path.exists(environment.vtroot + "/bin/memcached"):
args.extend(["-rowcache-bin", environment.vtroot + "/bin/memcached"])
else:
args.extend(["-rowcache-bin", "memcached"])
memcache_socket = os.path.join(self.tablet_dir, "memcache.sock")
args.extend(["-rowcache-socket", memcache_socket])
args.extend(["-enable-rowcache"])
if auth:
args.extend(['-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json')])
if customrules:
args.extend(['-customrules', customrules])
if schema_override:
args.extend(['-schema-override', schema_override])
if cert:
self.secure_port = environment.reserve_ports(1)
args.extend(['-secure-port', '%s' % self.secure_port,
'-cert', cert,
'-key', key])
if ca_cert:
args.extend(['-ca-cert', ca_cert])
stderr_fd = open(os.path.join(self.tablet_dir, "vttablet.stderr"), "w")
# increment count only the first time
if not self.proc:
Tablet.tablets_running += 1
self.proc = utils.run_bg(args, stderr=stderr_fd)
stderr_fd.close()
# wait for zookeeper PID just to be sure we have it
if environment.topo_server_implementation == 'zookeeper':
utils.run(environment.binary_path('zk')+' wait -e ' + self.zk_pid, stdout=utils.devnull)
# wait for query service to be in the right state
if wait_for_state:
self.wait_for_vttablet_state(wait_for_state, port=port)
return self.proc
def start(self):
args = environment.binary_args('vtctld') + [
'-debug',
'-templates', environment.vttop + '/go/cmd/vtctld/templates',
'-log_dir', environment.vtlogroot,
'-port', str(self.port),
] + \
environment.topo_server_flags() + \
environment.tablet_manager_protocol_flags()
stderr_fd = open(os.path.join(environment.tmproot, "vtctld.stderr"),
"w")
self.proc = run_bg(args, stderr=stderr_fd)
# wait for the process to listen to RPC
timeout = 30
while True:
v = get_vars(self.port)
if v:
break
timeout = wait_step('waiting for vtctld to start',
timeout,
sleep_time=0.2)
# save the running instance so vtctl commands can be remote executed now
global vtctld, vtctld_connection
if not vtctld:
vtctld = self
vtctld_connection = vtctl_client.connect(
environment.vtctl_client_protocol(),
'localhost:%u' % self.port, 30)
return self.proc
def run_vtworker(clargs,
log_level='',
auto_log=False,
expect_fail=False,
**kwargs):
args = environment.binary_args('vtworker') + [
'-log_dir', environment.vtlogroot, '-port',
str(environment.reserve_ports(1))
]
args.extend(environment.topo_server_flags())
args.extend(environment.tablet_manager_protocol_flags())
if auto_log:
if options.verbose == 2:
log_level = 'INFO'
elif options.verbose == 1:
log_level = 'WARNING'
else:
log_level = 'ERROR'
if log_level:
args.append('--stderrthreshold=%s' % log_level)
cmd = args + clargs
if expect_fail:
return run_fail(cmd, **kwargs)
return run(cmd, **kwargs)
def vtgate_start(vtport=None, cell='test_nj', retry_delay=1, retry_count=1, topo_impl=None, tablet_bson_encrypted=False, cache_ttl='1s', auth=False, timeout="5s", cert=None, key=None, ca_cert=None):
port = vtport or environment.reserve_ports(1)
secure_port = None
args = [environment.binary_path('vtgate'),
'-port', str(port),
'-cell', cell,
'-retry-delay', '%ss' % (str(retry_delay)),
'-retry-count', str(retry_count),
'-log_dir', environment.vtlogroot,
'-srv_topo_cache_ttl', cache_ttl,
'-timeout', timeout,
] + environment.tabletconn_protocol_flags()
if topo_impl:
args.extend(['-topo_implementation', topo_impl])
else:
args.extend(environment.topo_server_flags())
if tablet_bson_encrypted:
args.append('-tablet-bson-encrypted')
if auth:
args.extend(['-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json')])
if cert:
secure_port = environment.reserve_ports(1)
args.extend(['-secure-port', '%s' % secure_port,
'-cert', cert,
'-key', key])
if ca_cert:
args.extend(['-ca-cert', ca_cert])
sp = run_bg(args)
if cert:
wait_for_vars("vtgate", port, "SecureConnections")
return sp, port, secure_port
else:
wait_for_vars("vtgate", port)
return sp, port
def vtclient2(uid, path, query, bindvars=None, user=None, password=None, driver=None,
verbose=False, raise_on_error=True):
if (user is None) != (password is None):
raise TypeError("you should provide either both or none of user and password")
# for ZK paths to not have // in the path, that confuses things
if path.startswith('/'):
path = path[1:]
server = "localhost:%u/%s" % (uid, path)
cmdline = [environment.binary_path('vtclient2'), '-server', server]
cmdline += environment.topo_server_flags()
cmdline += environment.tabletconn_protocol_flags()
if user is not None:
cmdline.extend(['-tablet-bson-username', user,
'-tablet-bson-password', password])
if bindvars:
cmdline.extend(['-bindvars', bindvars])
if driver:
cmdline.extend(['-driver', driver])
if verbose:
cmdline.extend(['-alsologtostderr', '-verbose'])
cmdline.append(query)
return run(cmdline, raise_on_error=raise_on_error, trap_output=True)
def test_sigterm(self):
utils.run_vtctl('CreateKeyspace test_keyspace')
# create the database so vttablets start, as it is serving
tablet_62344.create_db('vt_test_keyspace')
tablet_62344.init_tablet('master', 'test_keyspace', '0', start=True)
# start a 'vtctl Sleep' command in the background
args = [
environment.binary_path('vtctl'), '-log_dir',
environment.vtlogroot, '--alsologtostderr'
]
args.extend(environment.topo_server_flags())
args.extend(environment.tablet_manager_protocol_flags())
args.extend(['Sleep', tablet_62344.tablet_alias, '60s'])
sp = utils.run_bg(args, stdout=PIPE, stderr=PIPE)
# wait for it to start, and let's kill it
time.sleep(4.0)
utils.run(['pkill', 'vtaction'])
out, err = sp.communicate()
# check the vtctl command got the right remote error back
if "vtaction interrupted by signal" not in err:
self.fail("cannot find expected output in error: " + err)
logging.debug("vtaction was interrupted correctly:\n" + err)
tablet_62344.kill_vttablet()
def run_vtctl(clargs, log_level='', auto_log=False, expect_fail=False, **kwargs):
args = environment.binary_args('vtctl') + ['-log_dir', environment.vtlogroot]
args.extend(environment.topo_server_flags())
args.extend(environment.tablet_manager_protocol_flags())
args.extend(environment.tabletconn_protocol_flags())
if auto_log:
if options.verbose == 2:
log_level='INFO'
elif options.verbose == 1:
log_level='WARNING'
else:
log_level='ERROR'
if log_level:
args.append('--stderrthreshold=%s' % log_level)
if isinstance(clargs, str):
cmd = " ".join(args) + ' ' + clargs
else:
cmd = args + clargs
if expect_fail:
return run_fail(cmd, **kwargs)
return run(cmd, **kwargs)
def run_vtctl(clargs, log_level='', auto_log=False, expect_fail=False, **kwargs):
args = [environment.binary_path('vtctl'), '-log_dir', environment.vtlogroot]
args.extend(environment.topo_server_flags())
args.extend(environment.tablet_manager_protocol_flags())
args.extend(environment.tabletconn_protocol_flags())
if auto_log:
if options.verbose == 2:
log_level='INFO'
elif options.verbose == 1:
log_level='WARNING'
else:
log_level='ERROR'
if log_level:
args.append('--stderrthreshold=%s' % log_level)
if isinstance(clargs, str):
cmd = " ".join(args) + ' ' + clargs
else:
cmd = args + clargs
if expect_fail:
return run_fail(cmd, **kwargs)
return run(cmd, **kwargs)
def vtclient2(uid, path, query, bindvars=None, user=None, password=None, driver=None,
verbose=False, raise_on_error=True):
if (user is None) != (password is None):
raise TypeError("you should provide either both or none of user and password")
# for ZK paths to not have // in the path, that confuses things
if path.startswith('/'):
path = path[1:]
server = "localhost:%u/%s" % (uid, path)
cmdline = environment.binary_args('vtclient2') + ['-server', server]
cmdline += environment.topo_server_flags()
cmdline += environment.tabletconn_protocol_flags()
if user is not None:
cmdline.extend(['-tablet-bson-username', user,
'-tablet-bson-password', password])
if bindvars:
cmdline.extend(['-bindvars', bindvars])
if driver:
cmdline.extend(['-driver', driver])
if verbose:
cmdline.extend(['-alsologtostderr', '-verbose'])
cmdline.append(query)
return run(cmdline, raise_on_error=raise_on_error, trap_output=True)
def start(self):
args = environment.binary_args('vtctld') + [
'-debug',
'-templates', environment.vttop + '/go/cmd/vtctld/templates',
'-log_dir', environment.vtlogroot,
'-port', str(self.port),
] + \
environment.topo_server_flags() + \
environment.tablet_manager_protocol_flags()
stderr_fd = open(os.path.join(environment.tmproot, "vtctld.stderr"), "w")
self.proc = run_bg(args, stderr=stderr_fd)
# wait for the process to listen to RPC
timeout = 30
while True:
v = get_vars(self.port)
if v:
break
timeout = wait_step('waiting for vtctld to start', timeout,
sleep_time=0.2)
# save the running instance so vtctl commands can be remote executed now
global vtctld, vtctld_connection
if not vtctld:
vtctld = self
vtctld_connection = vtctl_client.connect(
environment.vtctl_client_protocol(), 'localhost:%u' % self.port, 30)
return self.proc
def test_sigterm(self):
utils.run_vtctl('CreateKeyspace test_keyspace')
# create the database so vttablets start, as it is serving
tablet_62344.create_db('vt_test_keyspace')
tablet_62344.init_tablet('master', 'test_keyspace', '0', start=True)
# start a 'vtctl Sleep' command in the background
args = [environment.binary_path('vtctl'),
'-log_dir', environment.vtlogroot,
'--alsologtostderr']
args.extend(environment.topo_server_flags())
args.extend(environment.tablet_manager_protocol_flags())
args.extend(['Sleep', tablet_62344.tablet_alias, '60s'])
sp = utils.run_bg(args, stdout=PIPE, stderr=PIPE)
# wait for it to start, and let's kill it
time.sleep(4.0)
utils.run(['pkill', 'vtaction'])
out, err = sp.communicate()
# check the vtctl command got the right remote error back
if "vtaction interrupted by signal" not in err:
self.fail("cannot find expected output in error: " + err)
logging.debug("vtaction was interrupted correctly:\n" + err)
tablet_62344.kill_vttablet()
def test_actions_and_timeouts(self):
# Start up a master mysql and vttablet
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
tablet_62344.init_tablet('master', 'test_keyspace', '0')
utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
utils.validate_topology()
srvShard = utils.run_vtctl_json(['GetSrvShard', 'test_nj',
'test_keyspace/0'])
self.assertEqual(srvShard['MasterCell'], 'test_nj')
tablet_62344.create_db('vt_test_keyspace')
tablet_62344.start_vttablet()
utils.run_vtctl(['RpcPing', tablet_62344.tablet_alias])
# schedule long action in the background, sleep a little bit to make sure
# it started to run
args = (environment.binary_args('vtctl') +
environment.topo_server_flags() +
environment.tablet_manager_protocol_flags() +
environment.tabletconn_protocol_flags() +
['-log_dir', environment.vtlogroot,
'Sleep', tablet_62344.tablet_alias, '10s'])
bg = utils.run_bg(args)
time.sleep(3)
# try a frontend RpcPing that should timeout as the tablet is busy
# running the other one
stdout, stderr = utils.run_vtctl(['-wait-time', '3s',
'RpcPing', tablet_62344.tablet_alias],
expect_fail=True)
if 'Timeout waiting for' not in stderr:
self.fail("didn't find the right error strings in failed RpcPing: " +
stderr)
# wait for the background vtctl
bg.wait()
if environment.topo_server_implementation == 'zookeeper':
# extra small test: we ran for a while, get the states we were in,
# make sure they're accounted for properly
# first the query engine States
v = utils.get_vars(tablet_62344.port)
logging.debug("vars: %s" % str(v))
# then the Zookeeper connections
if v['ZkMetaConn']['test_nj']['Current'] != 'Connected':
self.fail('invalid zk test_nj state: %s' %
v['ZkMetaConn']['test_nj']['Current'])
if v['ZkMetaConn']['global']['Current'] != 'Connected':
self.fail('invalid zk global state: %s' %
v['ZkMetaConn']['global']['Current'])
if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9:
self.fail('not enough time in Connected state: %u',
v['ZkMetaConn']['test_nj']['DurationConnected'])
if v['TabletType'] != 'master':
self.fail('TabletType not exported correctly')
tablet_62344.kill_vttablet()
def test_actions_and_timeouts(self):
# Start up a master mysql and vttablet
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
tablet_62344.init_tablet('master', 'test_keyspace', '0')
utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
utils.validate_topology()
srvShard = utils.run_vtctl_json(
['GetSrvShard', 'test_nj', 'test_keyspace/0'])
self.assertEqual(srvShard['MasterCell'], 'test_nj')
tablet_62344.create_db('vt_test_keyspace')
tablet_62344.start_vttablet()
utils.run_vtctl(['Ping', tablet_62344.tablet_alias])
# schedule long action in the background, sleep a little bit to make sure
# it started to run
args = (environment.binary_args('vtctl') +
environment.topo_server_flags() +
environment.tablet_manager_protocol_flags() +
environment.tabletconn_protocol_flags() + [
'-log_dir', environment.vtlogroot, 'Sleep',
tablet_62344.tablet_alias, '10s'
])
bg = utils.run_bg(args)
time.sleep(3)
# try a frontend RefreshState that should timeout as the tablet is busy
# running the other one
stdout, stderr = utils.run_vtctl(
['-wait-time', '3s', 'RefreshState', tablet_62344.tablet_alias],
expect_fail=True)
self.assertIn(environment.rpc_timeout_message, stderr)
# wait for the background vtctl
bg.wait()
if environment.topo_server_implementation == 'zookeeper':
# extra small test: we ran for a while, get the states we were in,
# make sure they're accounted for properly
# first the query engine States
v = utils.get_vars(tablet_62344.port)
logging.debug("vars: %s" % str(v))
# then the Zookeeper connections
if v['ZkMetaConn']['test_nj']['Current'] != 'Connected':
self.fail('invalid zk test_nj state: %s' %
v['ZkMetaConn']['test_nj']['Current'])
if v['ZkMetaConn']['global']['Current'] != 'Connected':
self.fail('invalid zk global state: %s' %
v['ZkMetaConn']['global']['Current'])
if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9:
self.fail('not enough time in Connected state: %u',
v['ZkMetaConn']['test_nj']['DurationConnected'])
if v['TabletType'] != 'master':
self.fail('TabletType not exported correctly')
tablet_62344.kill_vttablet()
def vtgate_start(vtport=None,
cell='test_nj',
retry_delay=1,
retry_count=1,
topo_impl=None,
tablet_bson_encrypted=False,
cache_ttl='1s',
auth=False,
timeout="5s",
cert=None,
key=None,
ca_cert=None):
port = vtport or environment.reserve_ports(1)
secure_port = None
args = [
environment.binary_path('vtgate'),
'-port',
str(port),
'-cell',
cell,
'-retry-delay',
'%ss' % (str(retry_delay)),
'-retry-count',
str(retry_count),
'-log_dir',
environment.vtlogroot,
'-srv_topo_cache_ttl',
cache_ttl,
'-timeout',
timeout,
] + environment.tabletconn_protocol_flags()
if topo_impl:
args.extend(['-topo_implementation', topo_impl])
else:
args.extend(environment.topo_server_flags())
if tablet_bson_encrypted:
args.append('-tablet-bson-encrypted')
if auth:
args.extend([
'-auth-credentials',
os.path.join(environment.vttop, 'test', 'test_data',
'authcredentials_test.json')
])
if cert:
secure_port = environment.reserve_ports(1)
args.extend(
['-secure-port',
'%s' % secure_port, '-cert', cert, '-key', key])
if ca_cert:
args.extend(['-ca-cert', ca_cert])
sp = run_bg(args)
if cert:
wait_for_vars("vtgate", port, "SecureConnections")
return sp, port, secure_port
else:
wait_for_vars("vtgate", port)
return sp, port
def start(self):
args = [environment.binary_path('vtctld'),
'-debug',
'-templates', environment.vttop + '/go/cmd/vtctld/templates',
'-log_dir', environment.vtlogroot,
'-port', str(self.port),
] + \
environment.topo_server_flags() + \
environment.tablet_manager_protocol_flags()
stderr_fd = open(os.path.join(environment.tmproot, "vtctld.stderr"), "w")
self.proc = utils.run_bg(args, stderr=stderr_fd)
return self.proc
def start(self):
args = environment.binary_args('vtctld') + [
'-debug',
'-templates', environment.vttop + '/go/cmd/vtctld/templates',
'-log_dir', environment.vtlogroot,
'-port', str(self.port),
] + \
environment.topo_server_flags() + \
environment.tablet_manager_protocol_flags()
stderr_fd = open(os.path.join(environment.tmproot, "vtctld.stderr"),
"w")
self.proc = utils.run_bg(args, stderr=stderr_fd)
return self.proc
def vtgate_start(cell='test_nj', retry_delay=1, retry_count=1, topo_impl=None, tablet_bson_encrypted=False):
port = environment.reserve_ports(1)
args = [environment.binary_path('vtgate'),
'-port', str(port),
'-cell', cell,
'-retry-delay', '%ss' % (str(retry_delay)),
'-retry-count', str(retry_count),
'-log_dir', environment.vtlogroot,
] + environment.vtgate_protocol_flags()
if topo_impl:
args.extend(['-topo_implementation', topo_impl])
else:
args.extend(environment.topo_server_flags())
if tablet_bson_encrypted:
args.append('-tablet-bson-encrypted')
sp = run_bg(args)
wait_for_vars("vtgate", port)
return sp, port
def run_vtworker(clargs, log_level='', auto_log=False, expect_fail=False, **kwargs):
args = environment.binary_args('vtworker') + [
'-log_dir', environment.vtlogroot,
'-port', str(environment.reserve_ports(1))]
args.extend(environment.topo_server_flags())
args.extend(environment.tablet_manager_protocol_flags())
if auto_log:
if options.verbose == 2:
log_level='INFO'
elif options.verbose == 1:
log_level='WARNING'
else:
log_level='ERROR'
if log_level:
args.append('--stderrthreshold=%s' % log_level)
cmd = args + clargs
if expect_fail:
return run_fail(cmd, **kwargs)
return run(cmd, **kwargs)
def start_vttablet(self, port=None, auth=False, memcache=False,
wait_for_state="SERVING", customrules=None,
schema_override=None, cert=None, key=None, ca_cert=None,
repl_extra_flags={}, sensitive_mode=False,
target_tablet_type=None, lameduck_period=None,
extra_args=None, full_mycnf_args=False,
security_policy=None):
"""
Starts a vttablet process, and returns it.
The process is also saved in self.proc, so it's easy to kill as well.
"""
environment.prog_compile('vtaction')
args = [environment.binary_path('vttablet'),
'-port', '%s' % (port or self.port),
'-tablet-path', self.tablet_alias,
'-log_dir', environment.vtlogroot]
args.extend(environment.topo_server_flags())
args.extend(utils.binlog_player_protocol_flags)
dbconfigs = self._get_db_configs_file(repl_extra_flags)
for key1 in dbconfigs:
for key2 in dbconfigs[key1]:
args.extend(["-db-config-"+key1+"-"+key2, dbconfigs[key1][key2]])
if full_mycnf_args:
# this flag is used to specify all the mycnf_ flags, to make
# sure that code works and can fork actions.
relay_log_path = os.path.join(self.tablet_dir, "relay-logs",
"vt-%010d-relay-bin" % self.tablet_uid)
args.extend([
"-mycnf_server_id", str(self.tablet_uid),
"-mycnf_mysql_port", str(self.mysql_port),
"-mycnf_data_dir", os.path.join(self.tablet_dir, "data"),
"-mycnf_innodb_data_home_dir", os.path.join(self.tablet_dir,
"innodb", "data"),
"-mycnf_innodb_log_group_home_dir", os.path.join(self.tablet_dir,
"innodb", "logs"),
"-mycnf_socket_file", os.path.join(self.tablet_dir, "mysql.sock"),
"-mycnf_error_log_path", os.path.join(self.tablet_dir, "error.log"),
"-mycnf_slow_log_path", os.path.join(self.tablet_dir,
"slow-query.log"),
"-mycnf_relay_log_path", relay_log_path,
"-mycnf_relay_log_index_path", relay_log_path + ".index",
"-mycnf_relay_log_info_path", os.path.join(self.tablet_dir,
"relay-logs",
"relay-log.info"),
"-mycnf_bin_log_path", os.path.join(self.tablet_dir, "bin-logs",
"vt-%010d-bin" % self.tablet_uid),
"-mycnf_master_info_file", os.path.join(self.tablet_dir,
"master.info"),
"-mycnf_pid_file", os.path.join(self.tablet_dir, "mysql.pid"),
"-mycnf_tmp_dir", os.path.join(self.tablet_dir, "tmp"),
"-mycnf_slave_load_tmp_dir", os.path.join(self.tablet_dir, "tmp"),
])
if memcache:
args.extend(["-rowcache-bin", environment.memcached_bin()])
memcache_socket = os.path.join(self.tablet_dir, "memcache.sock")
args.extend(["-rowcache-socket", memcache_socket])
args.extend(["-enable-rowcache"])
if auth:
args.extend(['-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json')])
if customrules:
args.extend(['-customrules', customrules])
if schema_override:
args.extend(['-schema-override', schema_override])
if cert:
self.secure_port = environment.reserve_ports(1)
args.extend(['-secure-port', '%s' % self.secure_port,
'-cert', cert,
'-key', key])
if ca_cert:
args.extend(['-ca_cert', ca_cert])
if sensitive_mode:
args.extend(['-queryserver-config-sensitive-mode'])
if target_tablet_type:
args.extend(['-target_tablet_type', target_tablet_type,
'-health_check_interval', '2s',
'-allowed_replication_lag', '30'])
if lameduck_period:
args.extend(['-lameduck-period', lameduck_period])
if extra_args:
args.extend(extra_args)
if security_policy:
args.extend(['-security_policy', security_policy])
stderr_fd = open(os.path.join(self.tablet_dir, "vttablet.stderr"), "w")
# increment count only the first time
if not self.proc:
Tablet.tablets_running += 1
self.proc = utils.run_bg(args, stderr=stderr_fd)
stderr_fd.close()
# wait for zookeeper PID just to be sure we have it
if environment.topo_server_implementation == 'zookeeper':
utils.run(environment.binary_path('zk')+' wait -e ' + self.zk_pid, stdout=utils.devnull)
# wait for query service to be in the right state
if wait_for_state:
self.wait_for_vttablet_state(wait_for_state, port=port)
return self.proc
def test_reparent_down_master(self):
utils.run_vtctl('CreateKeyspace test_keyspace')
# create the database so vttablets start, as they are serving
tablet_62344.create_db('vt_test_keyspace')
tablet_62044.create_db('vt_test_keyspace')
tablet_41983.create_db('vt_test_keyspace')
tablet_31981.create_db('vt_test_keyspace')
# Start up a master mysql and vttablet
tablet_62344.init_tablet('master', 'test_keyspace', '0', start=True, wait_for_start=False)
# Create a few slaves for testing reparenting.
tablet_62044.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False)
tablet_41983.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False)
tablet_31981.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False)
# wait for all tablets to start
for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
t.wait_for_vttablet_state("SERVING")
# Recompute the shard layout node - until you do that, it might not be valid.
utils.run_vtctl('RebuildShardGraph test_keyspace/0')
utils.validate_topology()
# Force the slaves to reparent assuming that all the datasets are identical.
for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/0 ' + tablet_62344.tablet_alias, auto_log=True)
utils.validate_topology()
# Make the master agent and database unavailable.
tablet_62344.kill_vttablet()
tablet_62344.shutdown_mysql().wait()
self._check_db_addr('0', 'master', tablet_62344.port)
# Perform a reparent operation - the Validate part will try to ping
# the master and fail somewhat quickly
stdout, stderr = utils.run_vtctl('-wait-time 5s ReparentShard test_keyspace/0 ' + tablet_62044.tablet_alias, expect_fail=True)
logging.debug("Failed ReparentShard output:\n" + stderr)
if 'ValidateShard verification failed' not in stderr:
self.fail("didn't find the right error strings in failed ReparentShard: " + stderr)
# Should timeout and fail
stdout, stderr = utils.run_vtctl('-wait-time 5s ScrapTablet ' + tablet_62344.tablet_alias, expect_fail=True)
logging.debug("Failed ScrapTablet output:\n" + stderr)
if 'deadline exceeded' not in stderr:
self.fail("didn't find the right error strings in failed ScrapTablet: " + stderr)
# Should interrupt and fail
args = [environment.binary_path('vtctl'),
'-log_dir', environment.vtlogroot,
'-wait-time', '10s']
args.extend(environment.topo_server_flags())
args.extend(environment.tablet_manager_protocol_flags())
args.extend(['ScrapTablet', tablet_62344.tablet_alias])
sp = utils.run_bg(args, stdout=PIPE, stderr=PIPE)
# Need time for the process to start before killing it.
time.sleep(3.0)
os.kill(sp.pid, signal.SIGINT)
stdout, stderr = sp.communicate()
logging.debug("Failed ScrapTablet output:\n" + stderr)
if 'interrupted' not in stderr:
self.fail("didn't find the right error strings in failed ScrapTablet: " + stderr)
# Force the scrap action in zk even though tablet is not accessible.
tablet_62344.scrap(force=True)
utils.run_vtctl('ChangeSlaveType -force %s idle' %
tablet_62344.tablet_alias, expect_fail=True)
# Remove pending locks (make this the force option to ReparentShard?)
if environment.topo_server_implementation == 'zookeeper':
utils.run_vtctl('PurgeActions /zk/global/vt/keyspaces/test_keyspace/shards/0/action')
# Re-run reparent operation, this shoud now proceed unimpeded.
utils.run_vtctl('-wait-time 1m ReparentShard test_keyspace/0 ' + tablet_62044.tablet_alias, auto_log=True)
utils.validate_topology()
self._check_db_addr('0', 'master', tablet_62044.port)
utils.run_vtctl(['ChangeSlaveType', '-force', tablet_62344.tablet_alias, 'idle'])
idle_tablets, _ = utils.run_vtctl('ListAllTablets test_nj', trap_output=True)
if '0000062344 <null> <null> idle' not in idle_tablets:
self.fail('idle tablet not found: %s' % idle_tablets)
tablet.kill_tablets([tablet_62044, tablet_41983, tablet_31981])
# so the other tests don't have any surprise
tablet_62344.start_mysql().wait()
def start_vttablet(self, port=None, auth=False, memcache=False,
wait_for_state='SERVING', customrules=None,
schema_override=None, cert=None, key=None, ca_cert=None,
repl_extra_flags={}, table_acl_config=None,
lameduck_period=None, security_policy=None,
target_tablet_type=None, full_mycnf_args=False,
extra_args=None, extra_env=None, include_mysql_port=True):
"""Starts a vttablet process, and returns it.
The process is also saved in self.proc, so it's easy to kill as well.
"""
args = []
args.extend(['-tablet-path', self.tablet_alias])
args.extend(environment.topo_server_flags())
args.extend(utils.binlog_player_protocol_flags)
args.extend(environment.tablet_manager_protocol_flags())
args.extend(['-pid_file', os.path.join(self.tablet_dir, 'vttablet.pid')])
if self.use_mysqlctld:
args.extend(['-mysqlctl_socket', os.path.join(self.tablet_dir, 'mysqlctl.sock')])
if full_mycnf_args:
# this flag is used to specify all the mycnf_ flags, to make
# sure that code works and can fork actions.
relay_log_path = os.path.join(self.tablet_dir, 'relay-logs',
'vt-%010d-relay-bin' % self.tablet_uid)
args.extend([
'-mycnf_server_id', str(self.tablet_uid),
'-mycnf_data_dir', os.path.join(self.tablet_dir, 'data'),
'-mycnf_innodb_data_home_dir', os.path.join(self.tablet_dir,
'innodb', 'data'),
'-mycnf_innodb_log_group_home_dir', os.path.join(self.tablet_dir,
'innodb', 'logs'),
'-mycnf_socket_file', os.path.join(self.tablet_dir, 'mysql.sock'),
'-mycnf_error_log_path', os.path.join(self.tablet_dir, 'error.log'),
'-mycnf_slow_log_path', os.path.join(self.tablet_dir,
'slow-query.log'),
'-mycnf_relay_log_path', relay_log_path,
'-mycnf_relay_log_index_path', relay_log_path + '.index',
'-mycnf_relay_log_info_path', os.path.join(self.tablet_dir,
'relay-logs',
'relay-log.info'),
'-mycnf_bin_log_path', os.path.join(self.tablet_dir, 'bin-logs',
'vt-%010d-bin' % self.tablet_uid),
'-mycnf_master_info_file', os.path.join(self.tablet_dir,
'master.info'),
'-mycnf_pid_file', os.path.join(self.tablet_dir, 'mysql.pid'),
'-mycnf_tmp_dir', os.path.join(self.tablet_dir, 'tmp'),
'-mycnf_slave_load_tmp_dir', os.path.join(self.tablet_dir, 'tmp'),
])
if include_mysql_port:
args.extend(['-mycnf_mysql_port', str(self.mysql_port)])
if target_tablet_type:
args.extend(['-target_tablet_type', target_tablet_type,
'-health_check_interval', '2s',
'-allowed_replication_lag', '30'])
if extra_args:
args.extend(extra_args)
return self._start_prog(binary='vttablet', port=port, auth=auth,
memcache=memcache, wait_for_state=wait_for_state,
customrules=customrules,
schema_override=schema_override,
cert=cert, key=key, ca_cert=ca_cert,
repl_extra_flags=repl_extra_flags,
table_acl_config=table_acl_config,
lameduck_period=lameduck_period, extra_args=args,
security_policy=security_policy, extra_env=extra_env)
def start_vttablet(self,
port=None,
auth=False,
memcache=False,
wait_for_state="SERVING",
customrules=None,
schema_override=None,
cert=None,
key=None,
ca_cert=None,
repl_extra_flags={},
sensitive_mode=False,
target_tablet_type=None,
lameduck_period=None,
extra_args=None,
full_mycnf_args=False,
security_policy=None):
"""
Starts a vttablet process, and returns it.
The process is also saved in self.proc, so it's easy to kill as well.
"""
environment.prog_compile('vtaction')
args = [
environment.binary_path('vttablet'), '-port',
'%s' % (port or self.port), '-tablet-path', self.tablet_alias,
'-log_dir', environment.vtlogroot
]
args.extend(environment.topo_server_flags())
args.extend(utils.binlog_player_protocol_flags)
dbconfigs = self._get_db_configs_file(repl_extra_flags)
for key1 in dbconfigs:
for key2 in dbconfigs[key1]:
args.extend(
["-db-config-" + key1 + "-" + key2, dbconfigs[key1][key2]])
if full_mycnf_args:
# this flag is used to specify all the mycnf_ flags, to make
# sure that code works and can fork actions.
relay_log_path = os.path.join(
self.tablet_dir, "relay-logs",
"vt-%010d-relay-bin" % self.tablet_uid)
args.extend([
"-mycnf_server_id",
str(self.tablet_uid),
"-mycnf_mysql_port",
str(self.mysql_port),
"-mycnf_data_dir",
os.path.join(self.tablet_dir, "data"),
"-mycnf_innodb_data_home_dir",
os.path.join(self.tablet_dir, "innodb", "data"),
"-mycnf_innodb_log_group_home_dir",
os.path.join(self.tablet_dir, "innodb", "logs"),
"-mycnf_socket_file",
os.path.join(self.tablet_dir, "mysql.sock"),
"-mycnf_error_log_path",
os.path.join(self.tablet_dir, "error.log"),
"-mycnf_slow_log_path",
os.path.join(self.tablet_dir, "slow-query.log"),
"-mycnf_relay_log_path",
relay_log_path,
"-mycnf_relay_log_index_path",
relay_log_path + ".index",
"-mycnf_relay_log_info_path",
os.path.join(self.tablet_dir, "relay-logs", "relay-log.info"),
"-mycnf_bin_log_path",
os.path.join(self.tablet_dir, "bin-logs",
"vt-%010d-bin" % self.tablet_uid),
"-mycnf_master_info_file",
os.path.join(self.tablet_dir, "master.info"),
"-mycnf_pid_file",
os.path.join(self.tablet_dir, "mysql.pid"),
"-mycnf_tmp_dir",
os.path.join(self.tablet_dir, "tmp"),
"-mycnf_slave_load_tmp_dir",
os.path.join(self.tablet_dir, "tmp"),
])
if memcache:
args.extend(["-rowcache-bin", environment.memcached_bin()])
memcache_socket = os.path.join(self.tablet_dir, "memcache.sock")
args.extend(["-rowcache-socket", memcache_socket])
args.extend(["-enable-rowcache"])
if auth:
args.extend([
'-auth-credentials',
os.path.join(environment.vttop, 'test', 'test_data',
'authcredentials_test.json')
])
if customrules:
args.extend(['-customrules', customrules])
if schema_override:
args.extend(['-schema-override', schema_override])
if cert:
self.secure_port = environment.reserve_ports(1)
args.extend([
'-secure-port',
'%s' % self.secure_port, '-cert', cert, '-key', key
])
if ca_cert:
args.extend(['-ca_cert', ca_cert])
if sensitive_mode:
args.extend(['-queryserver-config-sensitive-mode'])
if target_tablet_type:
args.extend([
'-target_tablet_type', target_tablet_type,
'-health_check_interval', '2s', '-allowed_replication_lag',
'30'
])
if lameduck_period:
args.extend(['-lameduck-period', lameduck_period])
if extra_args:
args.extend(extra_args)
if security_policy:
args.extend(['-security_policy', security_policy])
stderr_fd = open(os.path.join(self.tablet_dir, "vttablet.stderr"), "w")
# increment count only the first time
if not self.proc:
Tablet.tablets_running += 1
self.proc = utils.run_bg(args, stderr=stderr_fd)
stderr_fd.close()
# wait for zookeeper PID just to be sure we have it
if environment.topo_server_implementation == 'zookeeper':
utils.run(environment.binary_path('zk') + ' wait -e ' +
self.zk_pid,
stdout=utils.devnull)
# wait for query service to be in the right state
if wait_for_state:
self.wait_for_vttablet_state(wait_for_state, port=port)
return self.proc
def start_vttablet(self, port=None, auth=False, memcache=False,
wait_for_state='SERVING', customrules=None,
schema_override=None, cert=None, key=None, ca_cert=None,
repl_extra_flags={}, table_acl_config=None,
target_tablet_type=None, lameduck_period=None,
extra_args=None, full_mycnf_args=False,
security_policy=None):
"""Starts a vttablet process, and returns it.
The process is also saved in self.proc, so it's easy to kill as well.
"""
environment.prog_compile('vtaction')
args = environment.binary_args('vttablet') + [
'-port', '%s' % (port or self.port),
'-tablet-path', self.tablet_alias,
'-log_dir', environment.vtlogroot]
args.extend(environment.topo_server_flags())
args.extend(utils.binlog_player_protocol_flags)
dbconfigs = self._get_db_configs_file(repl_extra_flags)
for key1 in dbconfigs:
for key2 in dbconfigs[key1]:
args.extend(['-db-config-' + key1 + '-' + key2, dbconfigs[key1][key2]])
if full_mycnf_args:
# this flag is used to specify all the mycnf_ flags, to make
# sure that code works and can fork actions.
relay_log_path = os.path.join(self.tablet_dir, 'relay-logs',
'vt-%010d-relay-bin' % self.tablet_uid)
args.extend([
'-mycnf_server_id', str(self.tablet_uid),
'-mycnf_mysql_port', str(self.mysql_port),
'-mycnf_data_dir', os.path.join(self.tablet_dir, 'data'),
'-mycnf_innodb_data_home_dir', os.path.join(self.tablet_dir,
'innodb', 'data'),
'-mycnf_innodb_log_group_home_dir', os.path.join(self.tablet_dir,
'innodb', 'logs'),
'-mycnf_socket_file', os.path.join(self.tablet_dir, 'mysql.sock'),
'-mycnf_error_log_path', os.path.join(self.tablet_dir, 'error.log'),
'-mycnf_slow_log_path', os.path.join(self.tablet_dir,
'slow-query.log'),
'-mycnf_relay_log_path', relay_log_path,
'-mycnf_relay_log_index_path', relay_log_path + '.index',
'-mycnf_relay_log_info_path', os.path.join(self.tablet_dir,
'relay-logs',
'relay-log.info'),
'-mycnf_bin_log_path', os.path.join(self.tablet_dir, 'bin-logs',
'vt-%010d-bin' % self.tablet_uid),
'-mycnf_master_info_file', os.path.join(self.tablet_dir,
'master.info'),
'-mycnf_pid_file', os.path.join(self.tablet_dir, 'mysql.pid'),
'-mycnf_tmp_dir', os.path.join(self.tablet_dir, 'tmp'),
'-mycnf_slave_load_tmp_dir', os.path.join(self.tablet_dir, 'tmp'),
])
if memcache:
args.extend(['-rowcache-bin', environment.memcached_bin()])
memcache_socket = os.path.join(self.tablet_dir, 'memcache.sock')
args.extend(['-rowcache-socket', memcache_socket])
args.extend(['-enable-rowcache'])
if auth:
args.extend(
['-auth-credentials',
os.path.join(
environment.vttop, 'test', 'test_data',
'authcredentials_test.json')])
if customrules:
args.extend(['-customrules', customrules])
if schema_override:
args.extend(['-schema-override', schema_override])
if table_acl_config:
args.extend(['-table-acl-config', table_acl_config])
args.extend(['-queryserver-config-strict-table-acl'])
if cert:
self.secure_port = environment.reserve_ports(1)
args.extend(['-secure-port', '%s' % self.secure_port,
'-cert', cert,
'-key', key])
if ca_cert:
args.extend(['-ca_cert', ca_cert])
if target_tablet_type:
args.extend(['-target_tablet_type', target_tablet_type,
'-health_check_interval', '2s',
'-allowed_replication_lag', '30'])
if lameduck_period:
args.extend(['-lameduck-period', lameduck_period])
if extra_args:
args.extend(extra_args)
if security_policy:
args.extend(['-security_policy', security_policy])
stderr_fd = open(os.path.join(self.tablet_dir, 'vttablet.stderr'), 'w')
# increment count only the first time
if not self.proc:
Tablet.tablets_running += 1
self.proc = utils.run_bg(args, stderr=stderr_fd)
stderr_fd.close()
# wait for query service to be in the right state
if wait_for_state:
self.wait_for_vttablet_state(wait_for_state, port=port)
return self.proc
def start_vttablet(self,
port=None,
auth=False,
memcache=False,
wait_for_state='SERVING',
customrules=None,
schema_override=None,
cert=None,
key=None,
ca_cert=None,
repl_extra_flags={},
table_acl_config=None,
lameduck_period=None,
security_policy=None,
target_tablet_type=None,
full_mycnf_args=False,
extra_args=None,
extra_env=None,
include_mysql_port=True):
"""Starts a vttablet process, and returns it.
The process is also saved in self.proc, so it's easy to kill as well.
"""
args = []
args.extend(['-tablet-path', self.tablet_alias])
args.extend(environment.topo_server_flags())
args.extend(utils.binlog_player_protocol_flags)
args.extend(environment.tablet_manager_protocol_flags())
args.extend(
['-pid_file',
os.path.join(self.tablet_dir, 'vttablet.pid')])
if full_mycnf_args:
# this flag is used to specify all the mycnf_ flags, to make
# sure that code works and can fork actions.
relay_log_path = os.path.join(
self.tablet_dir, 'relay-logs',
'vt-%010d-relay-bin' % self.tablet_uid)
args.extend([
'-mycnf_server_id',
str(self.tablet_uid),
'-mycnf_data_dir',
os.path.join(self.tablet_dir, 'data'),
'-mycnf_innodb_data_home_dir',
os.path.join(self.tablet_dir, 'innodb', 'data'),
'-mycnf_innodb_log_group_home_dir',
os.path.join(self.tablet_dir, 'innodb', 'logs'),
'-mycnf_socket_file',
os.path.join(self.tablet_dir, 'mysql.sock'),
'-mycnf_error_log_path',
os.path.join(self.tablet_dir, 'error.log'),
'-mycnf_slow_log_path',
os.path.join(self.tablet_dir, 'slow-query.log'),
'-mycnf_relay_log_path',
relay_log_path,
'-mycnf_relay_log_index_path',
relay_log_path + '.index',
'-mycnf_relay_log_info_path',
os.path.join(self.tablet_dir, 'relay-logs', 'relay-log.info'),
'-mycnf_bin_log_path',
os.path.join(self.tablet_dir, 'bin-logs',
'vt-%010d-bin' % self.tablet_uid),
'-mycnf_master_info_file',
os.path.join(self.tablet_dir, 'master.info'),
'-mycnf_pid_file',
os.path.join(self.tablet_dir, 'mysql.pid'),
'-mycnf_tmp_dir',
os.path.join(self.tablet_dir, 'tmp'),
'-mycnf_slave_load_tmp_dir',
os.path.join(self.tablet_dir, 'tmp'),
])
if include_mysql_port:
args.extend(['-mycnf_mysql_port', str(self.mysql_port)])
if target_tablet_type:
args.extend([
'-target_tablet_type', target_tablet_type,
'-health_check_interval', '2s', '-allowed_replication_lag',
'30'
])
if extra_args:
args.extend(extra_args)
return self._start_prog(binary='vttablet',
port=port,
auth=auth,
memcache=memcache,
wait_for_state=wait_for_state,
customrules=customrules,
schema_override=schema_override,
cert=cert,
key=key,
ca_cert=ca_cert,
repl_extra_flags=repl_extra_flags,
table_acl_config=table_acl_config,
lameduck_period=lameduck_period,
extra_args=args,
security_policy=security_policy,
extra_env=extra_env)
def test_reparent_down_master(self):
utils.run_vtctl('CreateKeyspace test_keyspace')
# create the database so vttablets start, as they are serving
tablet_62344.create_db('vt_test_keyspace')
tablet_62044.create_db('vt_test_keyspace')
tablet_41983.create_db('vt_test_keyspace')
tablet_31981.create_db('vt_test_keyspace')
# Start up a master mysql and vttablet
tablet_62344.init_tablet('master',
'test_keyspace',
'0',
start=True,
wait_for_start=False)
# Create a few slaves for testing reparenting.
tablet_62044.init_tablet('replica',
'test_keyspace',
'0',
start=True,
wait_for_start=False)
tablet_41983.init_tablet('replica',
'test_keyspace',
'0',
start=True,
wait_for_start=False)
tablet_31981.init_tablet('replica',
'test_keyspace',
'0',
start=True,
wait_for_start=False)
# wait for all tablets to start
for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
t.wait_for_vttablet_state("SERVING")
# Recompute the shard layout node - until you do that, it might not be valid.
utils.run_vtctl('RebuildShardGraph test_keyspace/0')
utils.validate_topology()
# Force the slaves to reparent assuming that all the datasets are identical.
for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/0 ' +
tablet_62344.tablet_alias,
auto_log=True)
utils.validate_topology()
# Make the master agent and database unavailable.
tablet_62344.kill_vttablet()
tablet_62344.shutdown_mysql().wait()
self._check_db_addr('0', 'master', tablet_62344.port)
# Perform a reparent operation - the Validate part will try to ping
# the master and fail somewhat quickly
stdout, stderr = utils.run_vtctl(
'-wait-time 5s ReparentShard test_keyspace/0 ' +
tablet_62044.tablet_alias,
expect_fail=True)
logging.debug("Failed ReparentShard output:\n" + stderr)
if 'ValidateShard verification failed' not in stderr:
self.fail(
"didn't find the right error strings in failed ReparentShard: "
+ stderr)
# Should timeout and fail
stdout, stderr = utils.run_vtctl('-wait-time 5s ScrapTablet ' +
tablet_62344.tablet_alias,
expect_fail=True)
logging.debug("Failed ScrapTablet output:\n" + stderr)
if 'deadline exceeded' not in stderr:
self.fail(
"didn't find the right error strings in failed ScrapTablet: " +
stderr)
# Should interrupt and fail
args = [
environment.binary_path('vtctl'), '-log_dir',
environment.vtlogroot, '-wait-time', '10s'
]
args.extend(environment.topo_server_flags())
args.extend(environment.tablet_manager_protocol_flags())
args.extend(['ScrapTablet', tablet_62344.tablet_alias])
sp = utils.run_bg(args, stdout=PIPE, stderr=PIPE)
# Need time for the process to start before killing it.
time.sleep(3.0)
os.kill(sp.pid, signal.SIGINT)
stdout, stderr = sp.communicate()
logging.debug("Failed ScrapTablet output:\n" + stderr)
if 'interrupted' not in stderr:
self.fail(
"didn't find the right error strings in failed ScrapTablet: " +
stderr)
# Force the scrap action in zk even though tablet is not accessible.
tablet_62344.scrap(force=True)
utils.run_vtctl('ChangeSlaveType -force %s idle' %
tablet_62344.tablet_alias,
expect_fail=True)
# Remove pending locks (make this the force option to ReparentShard?)
if environment.topo_server_implementation == 'zookeeper':
utils.run_vtctl(
'PurgeActions /zk/global/vt/keyspaces/test_keyspace/shards/0/action'
)
# Re-run reparent operation, this shoud now proceed unimpeded.
utils.run_vtctl('-wait-time 1m ReparentShard test_keyspace/0 ' +
tablet_62044.tablet_alias,
auto_log=True)
utils.validate_topology()
self._check_db_addr('0', 'master', tablet_62044.port)
utils.run_vtctl(
['ChangeSlaveType', '-force', tablet_62344.tablet_alias, 'idle'])
idle_tablets, _ = utils.run_vtctl('ListAllTablets test_nj',
trap_output=True)
if '0000062344 <null> <null> idle' not in idle_tablets:
self.fail('idle tablet not found: %s' % idle_tablets)
tablet.kill_tablets([tablet_62044, tablet_41983, tablet_31981])
# so the other tests don't have any surprise
tablet_62344.start_mysql().wait()