def setup(self, add_bad_host=False):
from environment import run, binary_args, vtlogroot, tmproot
self.assign_ports()
run(binary_args('zkctl') + [
'-log_dir', vtlogroot,
'-zk.cfg', '1@%s:%s' % (self.hostname, self.zk_ports),
'init'])
config = tmproot + '/test-zk-client-conf.json'
with open(config, 'w') as f:
ca_server = 'localhost:%u' % (self.zk_client_port)
if add_bad_host:
ca_server += ',does.not.exists:1234'
zk_cell_mapping = {
'test_nj': 'localhost:%u' % (self.zk_client_port),
'test_ny': 'localhost:%u' % (self.zk_client_port),
'test_ca': ca_server,
'global': 'localhost:%u' % (self.zk_client_port),
}
json.dump(zk_cell_mapping, f)
os.environ['ZK_CLIENT_CONFIG'] = config
logging.debug('Using ZK_CLIENT_CONFIG=%s', str(config))
run(binary_args('zk') + ['touch', '-p', '/zk/test_nj/vt'])
run(binary_args('zk') + ['touch', '-p', '/zk/test_ny/vt'])
run(binary_args('zk') + ['touch', '-p', '/zk/test_ca/vt'])
def setup(self, add_bad_host=False):
from environment import run, binary_args, vtlogroot # pylint: disable=g-import-not-at-top,g-multiple-import
import utils # pylint: disable=g-import-not-at-top
self.assign_ports()
run(binary_args('zkctl') + [
'-log_dir', vtlogroot,
'-zk.cfg', '1@%s:%s' % (self.hostname, self.zk_ports),
'init'])
# Create toplevel directories for global ZK, and one per cell.
run(binary_args('zk') + ['-server', self.addr, 'touch', '-p', '/global'])
run(binary_args('zk') + ['-server', self.addr, 'touch', '-p', '/test_nj'])
run(binary_args('zk') + ['-server', self.addr, 'touch', '-p', '/test_ny'])
run(binary_args('zk') + ['-server', self.addr, 'touch', '-p', '/test_ca'])
# Create the cell configurations using 'vtctl AddCellInfo'
utils.run_vtctl_vtctl(['AddCellInfo',
'-root', '/test_nj',
'-server_address', self.addr,
'test_nj'])
utils.run_vtctl_vtctl(['AddCellInfo',
'-root', '/test_ny',
'-server_address', self.addr,
'test_ny'])
ca_addr = self.addr
if add_bad_host:
ca_addr += ',does.not.exists:1234'
# Use UpdateCellInfo for this one, more coverage.
utils.run_vtctl_vtctl(['UpdateCellInfo',
'-root', '/test_ca',
'-server_address', ca_addr,
'test_ca'])
def wipe(self):
from environment import run, binary_args # pylint: disable=g-import-not-at-top,g-multiple-import
run(binary_args('zk') + ['-server', self.addr,
'rm', '-rf', '/zk/test_nj/vt/*'])
run(binary_args('zk') + ['-server', self.addr,
'rm', '-rf', '/zk/test_ny/vt/*'])
run(binary_args('zk') + ['-server', self.addr,
'rm', '-rf', '/zk/global/vt/*'])
def wipe(self):
from environment import run, binary_args # pylint: disable=g-import-not-at-top,g-multiple-import
# Only delete keyspaces/ in the global topology service, to keep
# the 'cells' directory. So we don't need to re-add the CellInfo records.
run(binary_args('zk') + ['-server', self.addr, 'rm', '-rf',
'/global/keyspaces'])
run(binary_args('zk') + ['-server', self.addr, 'rm', '-rf', '/test_nj/*'])
run(binary_args('zk') + ['-server', self.addr, 'rm', '-rf', '/test_ny/*'])
run(binary_args('zk') + ['-server', self.addr, 'rm', '-rf', '/test_ca/*'])
def wait_for_vttablet_state(self, expected, timeout=60.0, port=None):
# wait for zookeeper PID just to be sure we have it
if environment.topo_server_implementation == 'zookeeper':
if not self.checked_zk_pid:
utils.run(environment.binary_args('zk') + ['wait', '-e', self.zk_pid],
stdout=utils.devnull)
self.checked_zk_pid = True
while True:
v = utils.get_vars(port or self.port)
if v == None:
logging.debug(
' vttablet %s not answering at /debug/vars, waiting...',
self.tablet_alias)
else:
if 'Voltron' not in v:
logging.debug(
' vttablet %s not exporting Voltron, waiting...',
self.tablet_alias)
else:
s = v['TabletStateName']
if s != expected:
logging.debug(
' vttablet %s in state %s != %s', self.tablet_alias, s,
expected)
else:
break
timeout = utils.wait_step('waiting for state %s' % expected, timeout,
sleep_time=0.1)
def run_vtworker_client(args, rpc_port):
"""Runs vtworkerclient to execute a command on a remote vtworker.
Args:
args: Atr string to send to binary.
rpc_port: Port number.
Returns:
out: stdout of the vtworkerclient invocation
err: stderr of the vtworkerclient invocation
"""
out, err = run(
environment.binary_args("vtworkerclient")
+ [
"-vtworker_client_protocol",
protocols_flavor().vtworker_client_protocol(),
"-server",
"localhost:%d" % rpc_port,
"-stderrthreshold",
get_log_level(),
]
+ args,
trap_output=True,
)
return out, err
def mysqlctld(self, cmd, extra_my_cnf=None, verbose=False):
"""Runs a mysqlctld command.
Args:
cmd: the command to run.
extra_my_cnf: list of extra mycnf files to use
verbose: passed to mysqlctld.
Returns:
the result of run_bg.
"""
extra_env = {}
all_extra_my_cnf = get_all_extra_my_cnf(extra_my_cnf)
if all_extra_my_cnf:
extra_env["EXTRA_MY_CNF"] = ":".join(all_extra_my_cnf)
args = environment.binary_args("mysqlctld") + [
"-log_dir",
environment.vtlogroot,
"-tablet_uid",
str(self.tablet_uid),
"-mysql_port",
str(self.mysql_port),
"-socket_file",
os.path.join(self.tablet_dir, "mysqlctl.sock"),
]
self._add_dbconfigs(args)
if verbose:
args.append("-alsologtostderr")
args.extend(cmd)
return utils.run_bg(args, extra_env=extra_env)
def setUpModule():
global vtgateclienttest_process
global vtgateclienttest_port
global vtgateclienttest_grpc_port
try:
environment.topo_server().setup()
vtgateclienttest_port = environment.reserve_ports(1)
args = environment.binary_args('vtgateclienttest') + [
'-log_dir', environment.vtlogroot,
'-port', str(vtgateclienttest_port),
]
if protocols_flavor().vtgate_python_protocol() == 'grpc':
vtgateclienttest_grpc_port = environment.reserve_ports(1)
args.extend(['-grpc_port', str(vtgateclienttest_grpc_port)])
if protocols_flavor().service_map():
args.extend(['-service_map', ','.join(protocols_flavor().service_map())])
vtgateclienttest_process = utils.run_bg(args)
utils.wait_for_vars('vtgateclienttest', vtgateclienttest_port)
except:
tearDownModule()
raise
def run_vtctl_vtctl(clargs, auto_log=False, expect_fail=False,
**kwargs):
args = environment.binary_args('vtctl') + [
'-log_dir', environment.vtlogroot,
'-enable_queries',
]
args.extend(environment.topo_server().flags())
args.extend(['-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol()])
args.extend(['-tablet_protocol', protocols_flavor().tabletconn_protocol()])
args.extend(['-throttler_client_protocol',
protocols_flavor().throttler_client_protocol()])
args.extend(['-vtgate_protocol', protocols_flavor().vtgate_protocol()])
# TODO(b/26388813): Remove the next two lines once vtctl WaitForDrain is
# integrated in the vtctl MigrateServed* commands.
args.extend(['--wait_for_drain_sleep_rdonly', '0s'])
args.extend(['--wait_for_drain_sleep_replica', '0s'])
if auto_log:
args.append('--stderrthreshold=%s' % get_log_level())
if isinstance(clargs, str):
cmd = ' '.join(args) + ' ' + clargs
else:
cmd = args + clargs
if expect_fail:
return run_fail(cmd, **kwargs)
return run(cmd, **kwargs)
def mysqlctld(self, cmd, extra_my_cnf=None, verbose=False, extra_args=None):
"""Runs a mysqlctld command.
Args:
cmd: the command to run.
extra_my_cnf: list of extra mycnf files to use
verbose: passed to mysqlctld.
extra_args: passed to mysqlctld.
Returns:
the result of run_bg.
"""
extra_env = {}
all_extra_my_cnf = get_all_extra_my_cnf(extra_my_cnf)
if all_extra_my_cnf:
extra_env['EXTRA_MY_CNF'] = ':'.join(all_extra_my_cnf)
args = environment.binary_args('mysqlctld') + [
'-log_dir', environment.vtlogroot,
'-tablet_uid', str(self.tablet_uid),
'-mysql_port', str(self.mysql_port),
'-socket_file', os.path.join(self.tablet_dir, 'mysqlctl.sock')]
self._add_dbconfigs(self.default_db_dba_config, args)
if verbose:
args.append('-alsologtostderr')
if extra_args:
args.extend(extra_args)
args.extend(cmd)
return utils.run_bg(args, extra_env=extra_env)
def run_vtctl_vtctl(clargs, auto_log=False, expect_fail=False,
**kwargs):
args = environment.binary_args('vtctl') + [
'-log_dir', environment.vtlogroot,
'-enable_queries',
]
args.extend(environment.topo_server().flags())
args.extend(['-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol()])
args.extend(['-tablet_protocol', protocols_flavor().tabletconn_protocol()])
args.extend(['-throttler_client_protocol',
protocols_flavor().throttler_client_protocol()])
args.extend(['-vtgate_protocol', protocols_flavor().vtgate_protocol()])
if auto_log:
args.append('--stderrthreshold=%s' % get_log_level())
if isinstance(clargs, str):
cmd = ' '.join(args) + ' ' + clargs
else:
cmd = args + clargs
if expect_fail:
return run_fail(cmd, **kwargs)
return run(cmd, **kwargs)
def test_actions_and_timeouts(self):
# Start up a master mysql and vttablet
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
tablet_62344.init_tablet('master', 'test_keyspace', '0')
utils.validate_topology()
tablet_62344.create_db('vt_test_keyspace')
tablet_62344.start_vttablet()
utils.run_vtctl(['Ping', tablet_62344.tablet_alias])
# schedule long action in the background, sleep a little bit to make sure
# it started to run
args = (environment.binary_args('vtctl') +
environment.topo_server().flags() +
['-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol(),
'-tablet_protocol', protocols_flavor().tabletconn_protocol(),
'-log_dir', environment.vtlogroot,
'Sleep', tablet_62344.tablet_alias, '10s'])
bg = utils.run_bg(args)
time.sleep(3)
# try a frontend RefreshState that should timeout as the tablet is busy
# running the other one
_, stderr = utils.run_vtctl(
['-wait-time', '3s', 'RefreshState', tablet_62344.tablet_alias],
expect_fail=True)
self.assertIn(protocols_flavor().rpc_timeout_message(), stderr)
# wait for the background vtctl
bg.wait()
tablet_62344.kill_vttablet()
def vtclient(self, sql, keyspace=None, shard=None, tablet_type='master',
bindvars=None, streaming=False,
verbose=False, raise_on_error=True, json_output=False):
"""Uses the vtclient binary to send a query to vtgate."""
protocol, addr = self.rpc_endpoint()
args = environment.binary_args('vtclient') + [
'-server', addr,
'-tablet_type', tablet_type,
'-vtgate_protocol', protocol]
if json_output:
args.append('-json')
if keyspace:
args.extend(['-keyspace', keyspace])
if shard:
args.extend(['-shard', shard])
if bindvars:
args.extend(['-bind_variables', json.dumps(bindvars)])
if streaming:
args.append('-streaming')
if verbose:
args.append('-alsologtostderr')
args.append(sql)
out, err = run(args, raise_on_error=raise_on_error, trap_output=True)
if json_output:
return json.loads(out), err
return out, err
def tearDown(self):
self.preTeardown()
try:
mcu = self.mysql_conn.cursor()
for line in self.clean_sqls:
try:
mcu.execute(line, {})
except:
pass
mcu.close()
except:
pass
if getattr(self, "txlogger", None):
self.txlogger.terminate()
if getattr(self, "vtocc", None):
self.vtocc.terminate()
# stop mysql, delete directory
subprocess.call(environment.binary_args('mysqlctl') + [
"-tablet_uid", self.tabletuid,
"teardown", "-force"
])
try:
shutil.rmtree(self.mysqldir)
except:
pass
def start(self):
args = environment.binary_args('vtctld') + [
'-debug',
'-templates', environment.vttop + '/go/cmd/vtctld/templates',
'-log_dir', environment.vtlogroot,
'-port', str(self.port),
] + \
environment.topo_server_flags() + \
environment.tablet_manager_protocol_flags()
stderr_fd = open(os.path.join(environment.tmproot, "vtctld.stderr"), "w")
self.proc = run_bg(args, stderr=stderr_fd)
# wait for the process to listen to RPC
timeout = 30
while True:
v = get_vars(self.port)
if v:
break
timeout = wait_step('waiting for vtctld to start', timeout,
sleep_time=0.2)
# save the running instance so vtctl commands can be remote executed now
global vtctld, vtctld_connection
if not vtctld:
vtctld = self
vtctld_connection = vtctl_client.connect(
environment.vtctl_client_protocol(), 'localhost:%u' % self.port, 30)
return self.proc
def mysqlctl(self, cmd, extra_my_cnf=None, with_ports=False, verbose=False):
"""Runs a mysqlctl command.
Args:
cmd: the command to run.
extra_my_cnf: list of extra mycnf files to use
with_ports: if set, sends the tablet and mysql ports to mysqlctl.
verbose: passed to mysqlctld.
Returns:
the result of run_bg.
"""
extra_env = {}
all_extra_my_cnf = get_all_extra_my_cnf(extra_my_cnf)
if all_extra_my_cnf:
extra_env['EXTRA_MY_CNF'] = ':'.join(all_extra_my_cnf)
args = environment.binary_args('mysqlctl') + [
'-log_dir', environment.vtlogroot,
'-tablet_uid', str(self.tablet_uid)]
if self.use_mysqlctld:
args.extend(
['-mysqlctl_socket', os.path.join(self.tablet_dir, 'mysqlctl.sock')])
if with_ports:
args.extend(['-port', str(self.port),
'-mysql_port', str(self.mysql_port)])
self._add_dbconfigs(args)
if verbose:
args.append('-alsologtostderr')
args.extend(cmd)
return utils.run_bg(args, extra_env=extra_env)
def _get_vtworker_cmd(clargs, auto_log=False):
"""Assembles the command that is needed to run a vtworker.
Returns:
cmd - list of cmd arguments, can be passed to any `run`-like functions
port - int with the port number that the vtworker is running with
rpc_port - int with the port number of the RPC interface
"""
port = environment.reserve_ports(1)
rpc_port = port
args = environment.binary_args('vtworker') + [
'-log_dir', environment.vtlogroot,
'-min_healthy_rdonly_endpoints', '1',
'-port', str(port),
'-resolve_ttl', '2s',
'-executefetch_retry_time', '1s',
]
args.extend(environment.topo_server().flags())
args.extend(['-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol()])
if protocols_flavor().service_map():
args.extend(['-service_map',
",".join(protocols_flavor().service_map())])
if protocols_flavor().vtworker_client_protocol() == 'grpc':
rpc_port = environment.reserve_ports(1)
args.extend(['-grpc_port', str(rpc_port)])
if auto_log:
args.append('--stderrthreshold=%s' % get_log_level())
cmd = args + clargs
return cmd, port, rpc_port
def test_vtgate_qps(self):
# create the topology
utils.run_vtctl('CreateKeyspace test_keyspace')
t = tablet.Tablet(tablet_uid=1, cell="nj")
t.init_tablet("master", "test_keyspace", "0")
t.update_addrs()
utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)
# start vtgate and the qps-er
vtgate_proc, vtgate_port = utils.vtgate_start(
extra_args=['-cpu_profile', os.path.join(environment.tmproot,
'vtgate.pprof')])
qpser = utils.run_bg(environment.binary_args('zkclient2') + [
'-server', 'localhost:%u' % vtgate_port,
'-mode', 'qps',
'-zkclient_cpu_profile', os.path.join(environment.tmproot, 'zkclient2.pprof'),
'test_nj', 'test_keyspace'])
qpser.wait()
# get the vtgate vars, make sure we have what we need
v = utils.get_vars(vtgate_port)
# some checks on performance / stats
rpcCalls = v['TopoReaderRpcQueryCount']['test_nj']
if rpcCalls < MIN_QPS * 10:
self.fail('QPS is too low: %u < %u' % (rpcCalls / 10, MIN_QPS))
else:
logging.debug("Recorded qps: %u", rpcCalls / 10)
utils.vtgate_kill(vtgate_proc)
def run_vtctl_vtctl(clargs, log_level='', auto_log=False, expect_fail=False,
**kwargs):
args = environment.binary_args('vtctl') + ['-log_dir', environment.vtlogroot]
args.extend(environment.topo_server().flags())
args.extend(protocols_flavor().tablet_manager_protocol_flags())
args.extend(protocols_flavor().tabletconn_protocol_flags())
if auto_log:
if options.verbose == 2:
log_level='INFO'
elif options.verbose == 1:
log_level='WARNING'
else:
log_level='ERROR'
if log_level:
args.append('--stderrthreshold=%s' % log_level)
if isinstance(clargs, str):
cmd = " ".join(args) + ' ' + clargs
else:
cmd = args + clargs
if expect_fail:
return run_fail(cmd, **kwargs)
return run(cmd, **kwargs)
def vtclient2(uid, path, query, bindvars=None, user=None, password=None, driver=None,
verbose=False, raise_on_error=True):
if (user is None) != (password is None):
raise TypeError("you should provide either both or none of user and password")
# for ZK paths to not have // in the path, that confuses things
if path.startswith('/'):
path = path[1:]
server = "localhost:%u/%s" % (uid, path)
cmdline = environment.binary_args('vtclient2') + ['-server', server]
cmdline += environment.topo_server().flags()
cmdline += protocols_flavor().tabletconn_protocol_flags()
if user is not None:
cmdline.extend(['-tablet-bson-username', user,
'-tablet-bson-password', password])
if bindvars:
cmdline.extend(['-bindvars', bindvars])
if driver:
cmdline.extend(['-driver', driver])
if verbose:
cmdline.extend(['-alsologtostderr', '-verbose'])
cmdline.append(query)
return run(cmdline, raise_on_error=raise_on_error, trap_output=True)
def _get_vtworker_cmd(clargs, log_level='', auto_log=False):
"""Assembles the command that is needed to run a vtworker.
Returns:
cmd - list of cmd arguments, can be passed to any `run`-like functions
port - int with the port number that the vtworker is running with
"""
port = environment.reserve_ports(1)
args = environment.binary_args('vtworker') + [
'-log_dir', environment.vtlogroot,
'-min_healthy_rdonly_endpoints', '1',
'-port', str(port),
'-resolve_ttl', '2s',
'-executefetch_retry_time', '1s',
]
args.extend(environment.topo_server().flags())
args.extend(protocols_flavor().tablet_manager_protocol_flags())
if auto_log:
if options.verbose == 2:
log_level='INFO'
elif options.verbose == 1:
log_level='WARNING'
else:
log_level='ERROR'
if log_level:
args.append('--stderrthreshold=%s' % log_level)
cmd = args + clargs
return cmd, port
def setUpModule():
try:
environment.topo_server().setup()
logging.debug('Creating certificates')
os.makedirs(cert_dir)
utils.run(environment.binary_args('vttlstest') +
['-root', cert_dir,
'CreateCA'])
utils.run(environment.binary_args('vttlstest') +
['-root', cert_dir,
'CreateSignedCert',
'-common_name', 'Mysql Server',
'-serial', '01',
'server'])
utils.run(environment.binary_args('vttlstest') +
['-root', cert_dir,
'CreateSignedCert',
'-common_name', 'Mysql Client',
'-serial', '02',
'client'])
extra_my_cnf = cert_dir + '/secure.cnf'
fd = open(extra_my_cnf, 'w')
fd.write('ssl-ca=' + cert_dir + '/ca-cert.pem\n')
fd.write('ssl-cert=' + cert_dir + '/server-cert.pem\n')
fd.write('ssl-key=' + cert_dir + '/server-key.pem\n')
fd.close()
setup_procs = [
shard_0_master.init_mysql(extra_my_cnf=extra_my_cnf),
shard_0_slave.init_mysql(extra_my_cnf=extra_my_cnf),
]
utils.wait_procs(setup_procs)
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
shard_0_master.init_tablet('replica', 'test_keyspace', '0')
shard_0_slave.init_tablet('replica', 'test_keyspace', '0')
# create databases so vttablet can start behaving normally
shard_0_master.create_db('vt_test_keyspace')
shard_0_slave.create_db('vt_test_keyspace')
except:
tearDownModule()
raise
def test_actions_and_timeouts(self):
# Start up a master mysql and vttablet
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
tablet_62344.init_tablet('master', 'test_keyspace', '0')
utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
utils.validate_topology()
srvShard = utils.run_vtctl_json(['GetSrvShard', 'test_nj',
'test_keyspace/0'])
self.assertEqual(srvShard['MasterCell'], 'test_nj')
tablet_62344.create_db('vt_test_keyspace')
tablet_62344.start_vttablet()
utils.run_vtctl(['RpcPing', tablet_62344.tablet_alias])
# schedule long action in the background, sleep a little bit to make sure
# it started to run
args = (environment.binary_args('vtctl') +
environment.topo_server_flags() +
environment.tablet_manager_protocol_flags() +
environment.tabletconn_protocol_flags() +
['-log_dir', environment.vtlogroot,
'Sleep', tablet_62344.tablet_alias, '10s'])
bg = utils.run_bg(args)
time.sleep(3)
# try a frontend RpcPing that should timeout as the tablet is busy
# running the other one
stdout, stderr = utils.run_vtctl(['-wait-time', '3s',
'RpcPing', tablet_62344.tablet_alias],
expect_fail=True)
if 'Timeout waiting for' not in stderr:
self.fail("didn't find the right error strings in failed RpcPing: " +
stderr)
# wait for the background vtctl
bg.wait()
if environment.topo_server_implementation == 'zookeeper':
# extra small test: we ran for a while, get the states we were in,
# make sure they're accounted for properly
# first the query engine States
v = utils.get_vars(tablet_62344.port)
logging.debug("vars: %s" % str(v))
# then the Zookeeper connections
if v['ZkMetaConn']['test_nj']['Current'] != 'Connected':
self.fail('invalid zk test_nj state: %s' %
v['ZkMetaConn']['test_nj']['Current'])
if v['ZkMetaConn']['global']['Current'] != 'Connected':
self.fail('invalid zk global state: %s' %
v['ZkMetaConn']['global']['Current'])
if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9:
self.fail('not enough time in Connected state: %u',
v['ZkMetaConn']['test_nj']['DurationConnected'])
if v['TabletType'] != 'master':
self.fail('TabletType not exported correctly')
tablet_62344.kill_vttablet()
def test_primecache(self):
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
master.init_tablet( 'master', 'test_keyspace', '0')
replica.init_tablet('idle')
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
master.create_db('vt_test_keyspace')
master.start_vttablet(wait_for_state=None)
replica.start_vttablet(wait_for_state=None)
master.wait_for_vttablet_state('SERVING')
replica.wait_for_vttablet_state('NOT_SERVING') # DB doesn't exist
self._create_data()
# we use clone to not prime the mysql cache on the slave db
utils.run_vtctl(['Clone', '-force', '-server-mode',
master.tablet_alias, replica.tablet_alias],
auto_log=True)
# sync the buffer cache, and clear it. This will prompt for user's password
utils.run(['sync'])
utils.run(['sudo', 'bash', '-c', 'echo 1 > /proc/sys/vm/drop_caches'])
# we can now change data on the master for 30s, while slave is stopped.
# master's binlog will be in OS buffer cache now.
replica.mquery('', 'slave stop')
self._change_random_data()
use_primecache = True # easy to test without
if use_primecache:
# starting vtprimecache, sleeping for a couple seconds
args = environment.binary_args('vtprimecache') + [
'-db-config-dba-uname', 'vt_dba',
'-db-config-dba-charset', 'utf8',
'-db-config-dba-dbname', 'vt_test_keyspace',
'-db-config-app-uname', 'vt_app',
'-db-config-app-charset', 'utf8',
'-db-config-app-dbname', 'vt_test_keyspace',
'-relay_logs_path', replica.tablet_dir+'/relay-logs',
'-mysql_socket_file', replica.tablet_dir+'/mysql.sock',
'-log_dir', environment.vtlogroot,
'-worker_count', '4',
'-alsologtostderr',
]
vtprimecache = utils.run_bg(args)
time.sleep(2)
# start slave, see how longs it takes to catch up on replication
replica.mquery('', 'slave start')
self.catch_up()
if use_primecache:
# TODO(alainjobart): read and check stats
utils.kill_sub_process(vtprimecache)
tablet.kill_tablets([master, replica])
def wait_for_vttablet_state(self, expected, timeout=60.0, port=None):
# wait for zookeeper PID just to be sure we have it
if environment.topo_server().flavor() == 'zookeeper':
if not self.checked_zk_pid:
utils.run(environment.binary_args('zk') + ['wait', '-e', self.zk_pid],
stdout=utils.devnull)
self.checked_zk_pid = True
self.wait_for_vtocc_state(expected, timeout=timeout, port=port)
def zkocc_start(cells=['test_nj'], extra_params=[]):
args = environment.binary_args('zkocc') + [
'-port', str(environment.zkocc_port_base),
'-stderrthreshold=ERROR',
] + extra_params + cells
sp = run_bg(args)
wait_for_vars("zkocc", environment.zkocc_port_base)
return sp
def start(
self,
cell="test_nj",
retry_delay=1,
retry_count=2,
topo_impl=None,
cache_ttl="1s",
timeout_total="2s",
timeout_per_conn="1s",
extra_args=None,
tablets=None,
):
"""Start vtgate. Saves it into the global vtgate variable if not set yet."""
args = environment.binary_args("vtgate") + [
"-port",
str(self.port),
"-cell",
cell,
"-retry-delay",
"%ss" % (str(retry_delay)),
"-retry-count",
str(retry_count),
"-log_dir",
environment.vtlogroot,
"-srv_topo_cache_ttl",
cache_ttl,
"-conn-timeout-total",
timeout_total,
"-conn-timeout-per-conn",
timeout_per_conn,
"-bsonrpc_timeout",
"5s",
"-tablet_protocol",
protocols_flavor().tabletconn_protocol(),
"-gateway_implementation",
vtgate_gateway_flavor().flavor(),
"-tablet_types_to_wait",
"MASTER,REPLICA",
]
args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets))
if protocols_flavor().vtgate_protocol() == "grpc":
args.extend(["-grpc_port", str(self.grpc_port)])
if protocols_flavor().service_map():
args.extend(["-service_map", ",".join(protocols_flavor().service_map())])
if topo_impl:
args.extend(["-topo_implementation", topo_impl])
else:
args.extend(environment.topo_server().flags())
if extra_args:
args.extend(extra_args)
self.proc = run_bg(args)
wait_for_vars("vtgate", self.port)
global vtgate
if not vtgate:
vtgate = self
def start(
self,
cell="test_nj",
retry_delay=1,
retry_count=2,
topo_impl=None,
cache_ttl="1s",
timeout_total="4s",
timeout_per_conn="2s",
extra_args=None,
):
"""Starts the process for this vtgate instance.
If no other instance has been started, saves it into the global
vtgate variable.
"""
args = environment.binary_args("vtgate") + [
"-port",
str(self.port),
"-cell",
cell,
"-retry-delay",
"%ss" % (str(retry_delay)),
"-retry-count",
str(retry_count),
"-log_dir",
environment.vtlogroot,
"-srv_topo_cache_ttl",
cache_ttl,
"-conn-timeout-total",
timeout_total,
"-conn-timeout-per-conn",
timeout_per_conn,
"-bsonrpc_timeout",
"5s",
"-tablet_protocol",
protocols_flavor().tabletconn_protocol(),
]
if protocols_flavor().vtgate_protocol() == "grpc":
args.extend(["-grpc_port", str(self.grpc_port)])
if protocols_flavor().service_map():
args.extend(["-service_map", ",".join(protocols_flavor().service_map())])
if topo_impl:
args.extend(["-topo_implementation", topo_impl])
else:
args.extend(environment.topo_server().flags())
if extra_args:
args.extend(extra_args)
self.proc = run_bg(args)
if self.secure_port:
wait_for_vars("vtgate", self.port, "SecureConnections")
else:
wait_for_vars("vtgate", self.port)
global vtgate
if not vtgate:
vtgate = self
def start(self, enable_schema_change_dir=False):
# Note the vtctld2 web dir is set to 'dist', which is populated
# when a toplevel 'make build_web' is run. This is meant to test
# the development version of the UI. The real checked-in app is in
# app/.
args = environment.binary_args('vtctld') + [
'-enable_queries',
'-cell', 'test_nj',
'-web_dir', environment.vttop + '/web/vtctld',
'-web_dir2', environment.vttop + '/web/vtctld2/dist',
'--log_dir', environment.vtlogroot,
'--port', str(self.port),
'-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol(),
'-tablet_protocol', protocols_flavor().tabletconn_protocol(),
'-throttler_client_protocol',
protocols_flavor().throttler_client_protocol(),
'-vtgate_protocol', protocols_flavor().vtgate_protocol(),
'-workflow_manager_init',
'-workflow_manager_use_election',
] + environment.topo_server().flags()
# TODO(b/26388813): Remove the next two lines once vtctl WaitForDrain is
# integrated in the vtctl MigrateServed* commands.
args.extend(['--wait_for_drain_sleep_rdonly', '0s'])
args.extend(['--wait_for_drain_sleep_replica', '0s'])
if enable_schema_change_dir:
args += [
'--schema_change_dir', self.schema_change_dir,
'--schema_change_controller', 'local',
'--schema_change_check_interval', '1',
]
if protocols_flavor().service_map():
args.extend(['-service_map', ','.join(protocols_flavor().service_map())])
if protocols_flavor().vtctl_client_protocol() == 'grpc':
args.extend(['-grpc_port', str(self.grpc_port)])
stdout_fd = open(os.path.join(environment.tmproot, 'vtctld.stdout'), 'w')
stderr_fd = open(os.path.join(environment.tmproot, 'vtctld.stderr'), 'w')
self.proc = run_bg(args, stdout=stdout_fd, stderr=stderr_fd)
# wait for the process to listen to RPC
timeout = 30
while True:
v = get_vars(self.port)
if v:
break
if self.proc.poll() is not None:
raise TestError('vtctld died while starting')
timeout = wait_step('waiting for vtctld to start', timeout,
sleep_time=0.2)
# save the running instance so vtctl commands can be remote executed now
global vtctld, vtctld_connection
if not vtctld:
vtctld = self
protocol, endpoint = self.rpc_endpoint(python=True)
vtctld_connection = vtctl_client.connect(protocol, endpoint, 30)
return self.proc
def create_signed_cert(ca, serial, name, common_name):
logging.info('Creating signed cert and key %s', common_name)
utils.run(environment.binary_args('vttlstest') +
['-root', cert_dir,
'CreateSignedCert',
'-parent', ca,
'-serial', serial,
'-common_name', common_name,
name])
def start(self):
args = environment.binary_args('vtctld') + [
'-debug',
'-web_dir',
environment.vttop + '/web/vtctld',
'--templates',
environment.vttop + '/go/cmd/vtctld/templates',
'--log_dir',
environment.vtlogroot,
'--port',
str(self.port),
'--schema_change_dir',
self.schema_change_dir,
'--schema_change_controller',
'local',
'--schema_change_check_interval',
'1',
'-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol(),
'-vtgate_protocol',
protocols_flavor().vtgate_protocol(),
'-tablet_protocol',
protocols_flavor().tabletconn_protocol(),
] + environment.topo_server().flags()
if protocols_flavor().service_map():
args.extend(
['-service_map', ','.join(protocols_flavor().service_map())])
if protocols_flavor().vtctl_client_protocol() == 'grpc':
args.extend(['-grpc_port', str(self.grpc_port)])
stdout_fd = open(os.path.join(environment.tmproot, 'vtctld.stdout'),
'w')
stderr_fd = open(os.path.join(environment.tmproot, 'vtctld.stderr'),
'w')
self.proc = run_bg(args, stdout=stdout_fd, stderr=stderr_fd)
# wait for the process to listen to RPC
timeout = 30
while True:
v = get_vars(self.port)
if v:
break
timeout = wait_step('waiting for vtctld to start',
timeout,
sleep_time=0.2)
# save the running instance so vtctl commands can be remote executed now
global vtctld, vtctld_connection
if not vtctld:
vtctld = self
protocol, endpoint = self.rpc_endpoint(python=True)
vtctld_connection = vtctl_client.connect(protocol, endpoint, 30)
return self.proc
def test_actions_and_timeouts(self):
# Start up a master mysql and vttablet
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
tablet_62344.init_tablet('master', 'test_keyspace', '0')
utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
utils.validate_topology()
self._check_srv_shard()
tablet_62344.create_db('vt_test_keyspace')
tablet_62344.start_vttablet()
utils.run_vtctl(['Ping', tablet_62344.tablet_alias])
# schedule long action in the background, sleep a little bit to make sure
# it started to run
args = (environment.binary_args('vtctl') +
environment.topo_server().flags() +
['-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol(),
'-tablet_protocol', protocols_flavor().tabletconn_protocol(),
'-log_dir', environment.vtlogroot,
'Sleep', tablet_62344.tablet_alias, '10s'])
bg = utils.run_bg(args)
time.sleep(3)
# try a frontend RefreshState that should timeout as the tablet is busy
# running the other one
_, stderr = utils.run_vtctl(
['-wait-time', '3s', 'RefreshState', tablet_62344.tablet_alias],
expect_fail=True)
self.assertIn(protocols_flavor().rpc_timeout_message(), stderr)
# wait for the background vtctl
bg.wait()
if environment.topo_server().flavor() == 'zookeeper':
# extra small test: we ran for a while, get the states we were in,
# make sure they're accounted for properly
# first the query engine States
v = utils.get_vars(tablet_62344.port)
logging.debug('vars: %s', v)
# then the Zookeeper connections
if v['ZkCachedConn']['test_nj'] != 'Connected':
self.fail('invalid zk test_nj state: %s' %
v['ZkCachedConn']['test_nj'])
if v['ZkCachedConn']['global'] != 'Connected':
self.fail('invalid zk global state: %s' %
v['ZkCachedConn']['global'])
if v['TabletType'] != 'master':
self.fail('TabletType not exported correctly')
tablet_62344.kill_vttablet()
def start(self, cell='test_nj', retry_count=2,
topo_impl=None, cache_ttl='1s',
healthcheck_conn_timeout='2s',
extra_args=None, tablets=None,
tablet_types_to_wait='MASTER,REPLICA',
l2vtgates=None):
"""Start vtgate. Saves it into the global vtgate variable if not set yet."""
args = environment.binary_args('vtgate') + [
'-port', str(self.port),
'-cell', cell,
'-retry-count', str(retry_count),
'-log_dir', environment.vtlogroot,
'-srv_topo_cache_ttl', cache_ttl,
'-tablet_protocol', protocols_flavor().tabletconn_protocol(),
'-stderrthreshold', get_log_level(),
'-normalize_queries',
]
if l2vtgates:
args.extend([
'-gateway_implementation', 'l2vtgategateway',
'-l2vtgategateway_addrs', ','.join(l2vtgates),
])
else:
args.extend([
'-healthcheck_conn_timeout', healthcheck_conn_timeout,
'-gateway_implementation', vtgate_gateway_flavor().flavor(),
])
args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets))
if tablet_types_to_wait:
args.extend(['-tablet_types_to_wait', tablet_types_to_wait])
if protocols_flavor().vtgate_protocol() == 'grpc':
args.extend(['-grpc_port', str(self.grpc_port)])
args.extend(['-grpc_max_message_size', str(environment.grpc_max_message_size)])
if protocols_flavor().service_map():
args.extend(['-service_map', ','.join(protocols_flavor().service_map())])
if topo_impl:
args.extend(['-topo_implementation', topo_impl])
else:
args.extend(environment.topo_server().flags())
if extra_args:
args.extend(extra_args)
if self.mysql_port:
args.extend(['-mysql_server_port', str(self.mysql_port)])
self.proc = run_bg(args)
wait_for_vars('vtgate', self.port)
global vtgate
if not vtgate:
vtgate = self
def start(self):
args = environment.binary_args('vtctld') + [
'-debug',
'-templates', environment.vttop + '/go/cmd/vtctld/templates',
'-log_dir', environment.vtlogroot,
'-port', str(self.port),
] + \
environment.topo_server_flags() + \
environment.tablet_manager_protocol_flags()
stderr_fd = open(os.path.join(environment.tmproot, "vtctld.stderr"),
"w")
self.proc = utils.run_bg(args, stderr=stderr_fd)
return self.proc
def start(self,
cell='test_nj',
retry_count=2,
topo_impl=None,
cache_ttl='1s',
healthcheck_conn_timeout='2s',
extra_args=None,
tablets=None,
tablet_types_to_wait='MASTER,REPLICA'):
"""Start vtgate. Saves it into the global vtgate variable if not set yet."""
args = environment.binary_args('vtgate') + [
'-port',
str(self.port),
'-cell',
cell,
'-retry-count',
str(retry_count),
'-log_dir',
environment.vtlogroot,
'-srv_topo_cache_ttl',
cache_ttl,
'-healthcheck_conn_timeout',
healthcheck_conn_timeout,
'-tablet_protocol',
protocols_flavor().tabletconn_protocol(),
'-gateway_implementation',
vtgate_gateway_flavor().flavor(),
'-tablet_grpc_combine_begin_execute',
]
args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets))
if tablet_types_to_wait:
args.extend(['-tablet_types_to_wait', tablet_types_to_wait])
if protocols_flavor().vtgate_protocol() == 'grpc':
args.extend(['-grpc_port', str(self.grpc_port)])
if protocols_flavor().service_map():
args.extend(
['-service_map', ','.join(protocols_flavor().service_map())])
if topo_impl:
args.extend(['-topo_implementation', topo_impl])
else:
args.extend(environment.topo_server().flags())
if extra_args:
args.extend(extra_args)
self.proc = run_bg(args)
wait_for_vars('vtgate', self.port)
global vtgate
if not vtgate:
vtgate = self
def setUpModule():
try:
environment.topo_server().setup()
logging.debug('Creating certificates')
os.makedirs(cert_dir)
# Create CA certificate
utils.run(environment.binary_args('vttlstest') +
['-root', cert_dir,
'CreateCA'])
# create all certs
create_signed_cert('ca', '01', 'vttablet-server', 'vttablet server CA')
create_signed_cert('ca', '02', 'vttablet-client', 'vttablet client CA')
create_signed_cert('ca', '03', 'vtgate-server', 'vtgate server CA')
create_signed_cert('ca', '04', 'vtgate-client', 'vtgate client CA')
create_signed_cert('vttablet-server', '01', 'vttablet-server-instance',
'vttablet server instance')
create_signed_cert('vttablet-client', '01', 'vttablet-client-1',
'vttablet client 1')
create_signed_cert('vtgate-server', '01', 'vtgate-server-instance',
'localhost')
create_signed_cert('vtgate-client', '01', 'vtgate-client-1',
'vtgate client 1')
create_signed_cert('vtgate-client', '02', 'vtgate-client-2',
'vtgate client 2')
# setup all processes
setup_procs = [
shard_0_master.init_mysql(),
shard_0_slave.init_mysql(),
]
utils.wait_procs(setup_procs)
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
shard_0_master.init_tablet('replica', 'test_keyspace', '0')
shard_0_slave.init_tablet('replica', 'test_keyspace', '0')
# create databases so vttablet can start behaving normally
shard_0_master.create_db('vt_test_keyspace')
shard_0_slave.create_db('vt_test_keyspace')
except:
tearDownModule()
raise
def run_vtworker_client(args, rpc_port):
"""Runs vtworkerclient to execute a command on a remote vtworker.
Returns:
out - stdout of the vtworkerclient invocation
err - stderr of the vtworkerclient invocation
"""
out, err = run(environment.binary_args('vtworkerclient') +
['-vtworker_client_protocol',
protocols_flavor().vtworker_client_protocol(),
'-server', 'localhost:%d' % rpc_port,
'-stderrthreshold', get_log_level()] + args,
trap_output=True)
return out, err
def mysqlctld(self, cmd, extra_my_cnf=None, with_ports=False, verbose=False):
extra_env = {}
all_extra_my_cnf = get_all_extra_my_cnf(extra_my_cnf)
if all_extra_my_cnf:
extra_env['EXTRA_MY_CNF'] = ':'.join(all_extra_my_cnf)
args = environment.binary_args('mysqlctld') + [
'-log_dir', environment.vtlogroot,
'-tablet_uid', str(self.tablet_uid),
'-mysql_port', str(self.mysql_port),
'-socket_file', os.path.join(self.tablet_dir, 'mysqlctl.sock')]
if verbose:
args.append('-alsologtostderr')
args.extend(cmd)
return utils.run_bg(args, extra_env=extra_env)
def start(self,
cell='test_nj',
retry_count=2,
topo_impl=None,
cache_ttl='1s',
extra_args=None,
tablets=None,
tablet_types_to_wait='MASTER,REPLICA',
tablet_filters=None):
"""Start l2vtgate."""
args = environment.binary_args('l2vtgate') + [
'-port',
str(self.port),
'-cell',
cell,
'-retry-count',
str(retry_count),
'-log_dir',
environment.vtlogroot,
'-srv_topo_cache_ttl',
cache_ttl,
'-srv_topo_cache_refresh',
cache_ttl,
'-tablet_protocol',
protocols_flavor().tabletconn_protocol(),
'-gateway_implementation',
vtgate_gateway_flavor().flavor(),
]
args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets))
if tablet_types_to_wait:
args.extend(['-tablet_types_to_wait', tablet_types_to_wait])
if tablet_filters:
args.extend(['-tablet_filters', tablet_filters])
if protocols_flavor().vtgate_protocol() == 'grpc':
args.extend(['-grpc_port', str(self.grpc_port)])
if protocols_flavor().service_map():
args.extend(
['-service_map', ','.join(protocols_flavor().service_map())])
if topo_impl:
args.extend(['-topo_implementation', topo_impl])
else:
args.extend(environment.topo_server().flags())
if extra_args:
args.extend(extra_args)
self.proc = run_bg(args)
# We use a longer timeout here, as we may be waiting for the initial
# state of a few tablets.
wait_for_vars('l2vtgate', self.port, timeout=20.0)
def vtctl_client(self, args):
if options.verbose == 2:
log_level = 'INFO'
elif options.verbose == 1:
log_level = 'WARNING'
else:
log_level = 'ERROR'
out, err = run(environment.binary_args('vtctlclient') + [
'-vtctl_client_protocol',
protocols_flavor().vtctl_client_protocol(), '-server',
'localhost:%u' % self.port, '-stderrthreshold', log_level
] + args,
trap_output=True)
return out
def vtctl_client(self, args):
if options.verbose == 2:
log_level = 'INFO'
elif options.verbose == 1:
log_level = 'WARNING'
else:
log_level = 'ERROR'
protocol, endpoint = self.rpc_endpoint()
out, _ = run(environment.binary_args('vtctlclient') + [
'-vtctl_client_protocol', protocol, '-server', endpoint,
'-stderrthreshold', log_level
] + args,
trap_output=True)
return out
def run_vtworker_client_bg(args, rpc_port):
"""Runs vtworkerclient to execute a command on a remote vtworker.
Args:
args: Full vtworker command.
rpc_port: Port number.
Returns:
proc: process returned by subprocess.Popen
"""
return run_bg(
environment.binary_args('vtworkerclient') +
['-vtworker_client_protocol',
protocols_flavor().vtworker_client_protocol(),
'-server', 'localhost:%d' % rpc_port,
'-stderrthreshold', get_log_level()] + args)
def _get_vtworker_cmd(clargs, auto_log=False):
"""Assembles the command that is needed to run a vtworker.
Args:
clargs: Command line arguments passed to vtworker.
auto_log: If true, set --stderrthreshold according to the test log level.
Returns:
cmd - list of cmd arguments, can be passed to any `run`-like functions
port - int with the port number that the vtworker is running with
rpc_port - int with the port number of the RPC interface
"""
port = environment.reserve_ports(1)
rpc_port = port
args = environment.binary_args('vtworker') + [
'-log_dir',
environment.vtlogroot,
'-min_healthy_rdonly_endpoints',
'1',
'-port',
str(port),
# use a long resolve TTL because of potential race conditions with doing
# an EmergencyReparent and resolving the master (as EmergencyReparent
# will delete the old master before updating the shard record with the
# new master)
'-resolve_ttl',
'10s',
'-executefetch_retry_time',
'1s',
'-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol(),
'-tablet_protocol',
protocols_flavor().tabletconn_protocol(),
]
args.extend(environment.topo_server().flags())
if protocols_flavor().service_map():
args.extend(
['-service_map', ','.join(protocols_flavor().service_map())])
if protocols_flavor().vtworker_client_protocol() == 'grpc':
rpc_port = environment.reserve_ports(1)
args.extend(['-grpc_port', str(rpc_port)])
if auto_log:
args.append('--stderrthreshold=%s' % get_log_level())
cmd = args + clargs
return cmd, port, rpc_port
def test_vertical_split(self):
# Use a dedicated worker to run all vtworker commands.
worker_proc, _, worker_rpc_port = utils.run_vtworker_bg(
['--cell', 'test_nj'], auto_log=True)
vtworker_endpoint = 'localhost:' + str(worker_rpc_port)
automation_server_proc, automation_server_port = (
utils.run_automation_server())
_, vtctld_endpoint = utils.vtctld.rpc_endpoint()
params = {
'source_keyspace': 'source_keyspace',
'dest_keyspace': 'destination_keyspace',
'shard_list': '0',
'tables': 'moving.*,view1',
'vtctld_endpoint': vtctld_endpoint,
'vtworker_endpoint': vtworker_endpoint,
}
args = [
'--server', 'localhost:' + str(automation_server_port), '--task',
'VerticalSplitTask'
]
args.extend(['--param=' + k + '=' + v for k, v in params.items()])
utils.run(environment.binary_args('automation_client') + args)
# One of the two source rdonly tablets went spare after the diff.
# Force a healthcheck on both to get them back to "rdonly".
for t in [
vertical_split.source_rdonly1, vertical_split.source_rdonly2
]:
utils.run_vtctl(['RunHealthCheck', t.tablet_alias])
self._check_srv_keyspace('')
self._check_blacklisted_tables(vertical_split.source_master,
['moving.*', 'view1'])
self._check_blacklisted_tables(vertical_split.source_replica,
['moving.*', 'view1'])
self._check_blacklisted_tables(vertical_split.source_rdonly1,
['moving.*', 'view1'])
self._check_blacklisted_tables(vertical_split.source_rdonly2,
['moving.*', 'view1'])
# check the binlog player is gone now
vertical_split.destination_master.wait_for_binlog_player_count(0)
utils.kill_sub_process(automation_server_proc, soft=True)
utils.kill_sub_process(worker_proc, soft=True)
def start(self):
args = environment.binary_args('vtctld') + [
'-debug',
'--templates', environment.vttop + '/go/cmd/vtctld/templates',
'--log_dir', environment.vtlogroot,
'--port', str(self.port),
'--schema_change_dir', self.schema_change_dir,
'--schema_change_controller', 'local',
'--schema_change_check_interval', '1',
'-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol(),
] + \
environment.topo_server().flags() + \
protocols_flavor().vtgate_protocol_flags()
if protocols_flavor().service_map():
args.extend(
['-service_map', ",".join(protocols_flavor().service_map())])
if protocols_flavor().vtctl_client_protocol() == 'grpc':
args.extend(['-grpc_port', str(self.grpc_port)])
stderr_fd = open(os.path.join(environment.tmproot, 'vtctld.stderr'),
'w')
self.proc = run_bg(args, stderr=stderr_fd)
# wait for the process to listen to RPC
timeout = 30
while True:
v = get_vars(self.port)
if v:
break
timeout = wait_step('waiting for vtctld to start',
timeout,
sleep_time=0.2)
# save the running instance so vtctl commands can be remote executed now
protocol = protocols_flavor().vtctl_client_protocol()
if protocol == "grpc":
# import the grpc vtctl client implementation, disabled for now:
# from vtctl import grpc_vtctl_client
# temporary protocol override until python client support works
protocol = "gorpc"
global vtctld, vtctld_connection
if not vtctld:
vtctld = self
vtctld_connection = vtctl_client.connect(
protocol, 'localhost:%u' % self.port, 30)
return self.proc
def wipe(self):
from environment import run, binary_args # pylint: disable=g-import-not-at-top,g-multiple-import
# Work around safety check on recursive delete.
run(binary_args('zk') + ['rm', '-rf', '/zk/test_nj/vt/*'])
run(binary_args('zk') + ['rm', '-rf', '/zk/test_ny/vt/*'])
run(binary_args('zk') + ['rm', '-rf', '/zk/global/vt/*'])
run(binary_args('zk') + ['rm', '-f', '/zk/test_nj/vt'])
run(binary_args('zk') + ['rm', '-f', '/zk/test_ny/vt'])
run(binary_args('zk') + ['rm', '-f', '/zk/global/vt'])
def wipe(self):
from environment import run, binary_args
# Work around safety check on recursive delete.
run(binary_args('zk') + ['rm', '-rf', '/zk/test_nj/vt/*'])
run(binary_args('zk') + ['rm', '-rf', '/zk/test_ny/vt/*'])
run(binary_args('zk') + ['rm', '-rf', '/zk/global/vt/*'])
run(binary_args('zk') + ['rm', '-f', '/zk/test_nj/vt'])
run(binary_args('zk') + ['rm', '-f', '/zk/test_ny/vt'])
run(binary_args('zk') + ['rm', '-f', '/zk/global/vt'])
def test_actions_and_timeouts(self):
# Start up a master mysql and vttablet
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
tablet_62344.init_tablet('master', 'test_keyspace', '0')
tablet_62344.create_db('vt_test_keyspace')
tablet_62344.start_vttablet()
# validate topology after starting tablet so that tablet has a chance
# to update shard master_alias
timeout = 10
while True:
shard = utils.run_vtctl_json(['GetShard', 'test_keyspace/0'])
if shard['master_alias']['uid'] == 62344:
break
wait_step('master_alias has been set', timeout)
utils.validate_topology()
utils.run_vtctl(['Ping', tablet_62344.tablet_alias])
# schedule long action in the background, sleep a little bit to make sure
# it started to run
args = (environment.binary_args('vtctl') +
environment.topo_server().flags() + [
'-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol(),
'-tablet_protocol',
protocols_flavor().tabletconn_protocol(), '-log_dir',
environment.vtlogroot, 'Sleep', tablet_62344.tablet_alias,
'10s'
])
bg = utils.run_bg(args)
time.sleep(3)
# try a frontend RefreshState that should timeout as the tablet is busy
# running the other one
_, stderr = utils.run_vtctl(
['-wait-time', '3s', 'RefreshState', tablet_62344.tablet_alias],
expect_fail=True)
self.assertIn(protocols_flavor().rpc_timeout_message(), stderr)
# wait for the background vtctl
bg.wait()
tablet_62344.kill_vttablet()
def start(self, cell='test_nj', retry_delay=1, retry_count=2,
topo_impl=None, cache_ttl='1s',
auth=False, timeout_total='4s', timeout_per_conn='2s',
extra_args=None):
"""Starts the process for this vtgate instance.
If no other instance has been started, saves it into the global
vtgate variable.
"""
args = environment.binary_args('vtgate') + [
'-port', str(self.port),
'-cell', cell,
'-retry-delay', '%ss' % (str(retry_delay)),
'-retry-count', str(retry_count),
'-log_dir', environment.vtlogroot,
'-srv_topo_cache_ttl', cache_ttl,
'-conn-timeout-total', timeout_total,
'-conn-timeout-per-conn', timeout_per_conn,
'-bsonrpc_timeout', '5s',
'-tablet_protocol', protocols_flavor().tabletconn_protocol(),
]
if protocols_flavor().vtgate_protocol() == 'grpc':
args.extend(['-grpc_port', str(self.grpc_port)])
if protocols_flavor().service_map():
args.extend(['-service_map', ','.join(protocols_flavor().service_map())])
if topo_impl:
args.extend(['-topo_implementation', topo_impl])
else:
args.extend(environment.topo_server().flags())
if auth:
args.extend(['-auth-credentials',
os.path.join(environment.vttop, 'test', 'test_data',
'authcredentials_test.json')])
if extra_args:
args.extend(extra_args)
self.proc = run_bg(args)
if self.secure_port:
wait_for_vars('vtgate', self.port, 'SecureConnections')
else:
wait_for_vars('vtgate', self.port)
global vtgate
if not vtgate:
vtgate = self
def run_automation_server(auto_log=False):
"""Starts a background automation_server process.
Returns:
rpc_port - int with the port number of the RPC interface
"""
rpc_port = environment.reserve_ports(1)
args = environment.binary_args('automation_server') + [
'-log_dir', environment.vtlogroot,
'-port', str(rpc_port),
'-vtctl_client_protocol', protocols_flavor().vtctl_client_protocol(),
'-vtworker_client_protocol',
protocols_flavor().vtworker_client_protocol(),
]
if auto_log:
args.append('--stderrthreshold=%s' % get_log_level())
return run_bg(args), rpc_port
def _get_vtworker_cmd(clargs, log_level='', auto_log=False):
"""Assembles the command that is needed to run a vtworker.
Returns:
cmd - list of cmd arguments, can be passed to any `run`-like functions
port - int with the port number that the vtworker is running with
"""
port = environment.reserve_ports(1)
rpc_port = environment.reserve_ports(1)
args = environment.binary_args('vtworker') + [
'-log_dir',
environment.vtlogroot,
'-min_healthy_rdonly_endpoints',
'1',
'-port',
str(port),
'-resolve_ttl',
'2s',
'-executefetch_retry_time',
'1s',
]
args.extend(environment.topo_server().flags())
args.extend([
'-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol()
])
if protocols_flavor().service_map():
args.extend(
['-service_map', ",".join(protocols_flavor().service_map())])
if protocols_flavor().vtworker_client_protocol() == 'grpc':
args.extend(['-grpc_port', str(rpc_port)])
if auto_log:
if options.verbose == 2:
log_level = 'INFO'
elif options.verbose == 1:
log_level = 'WARNING'
else:
log_level = 'ERROR'
if log_level:
args.append('--stderrthreshold=%s' % log_level)
cmd = args + clargs
return cmd, port, rpc_port
def vtclient(self, sql, tablet_type='master', bindvars=None,
streaming=False, verbose=False, raise_on_error=False):
"""vtclient uses the vtclient binary to send a query to vtgate.
"""
args = environment.binary_args('vtclient') + [
'-server', self.rpc_endpoint(),
'-tablet_type', tablet_type,
'-vtgate_protocol', protocols_flavor().vtgate_protocol()]
if bindvars:
args.extend(['-bind_variables', json.dumps(bindvars)])
if streaming:
args.append('-streaming')
if verbose:
args.append('-alsologtostderr')
args.append(sql)
out, err = run(args, raise_on_error=raise_on_error, trap_output=True)
out = out.splitlines()
return out, err
def mysqlctl(self,
cmd,
extra_my_cnf=None,
with_ports=False,
verbose=False,
extra_args=None):
"""Runs a mysqlctl command.
Args:
cmd: the command to run.
extra_my_cnf: list of extra mycnf files to use
with_ports: if set, sends the tablet and mysql ports to mysqlctl.
verbose: passed to mysqlctld.
extra_args: passed to mysqlctl.
Returns:
the result of run_bg.
"""
extra_env = {}
all_extra_my_cnf = get_all_extra_my_cnf(extra_my_cnf)
if all_extra_my_cnf:
extra_env['EXTRA_MY_CNF'] = ':'.join(all_extra_my_cnf)
args = environment.binary_args('mysqlctl') + [
'-log_dir', environment.vtlogroot, '-tablet_uid',
str(self.tablet_uid)
]
if self.use_mysqlctld:
args.extend([
'-mysqlctl_socket',
os.path.join(self.tablet_dir, 'mysqlctl.sock')
])
if with_ports:
args.extend(
['-port',
str(self.port), '-mysql_port',
str(self.mysql_port)])
self._add_dbconfigs(self.default_db_dba_config, args)
if verbose:
args.append('-alsologtostderr')
if extra_args:
args.extend(extra_args)
args.extend(cmd)
return utils.run_bg(args, extra_env=extra_env)
def vtgate_start(vtport=None, cell='test_nj', retry_delay=1, retry_count=1,
topo_impl=None, tablet_bson_encrypted=False, cache_ttl='1s',
auth=False, timeout="5s", cert=None, key=None, ca_cert=None,
socket_file=None, extra_args=None):
port = vtport or environment.reserve_ports(1)
secure_port = None
args = environment.binary_args('vtgate') + [
'-port', str(port),
'-cell', cell,
'-retry-delay', '%ss' % (str(retry_delay)),
'-retry-count', str(retry_count),
'-log_dir', environment.vtlogroot,
'-srv_topo_cache_ttl', cache_ttl,
'-timeout', timeout,
] + environment.tabletconn_protocol_flags()
if topo_impl:
args.extend(['-topo_implementation', topo_impl])
else:
args.extend(environment.topo_server_flags())
if tablet_bson_encrypted:
args.append('-tablet-bson-encrypted')
if auth:
args.extend(['-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json')])
if cert:
secure_port = environment.reserve_ports(1)
args.extend(['-secure-port', '%s' % secure_port,
'-cert', cert,
'-key', key])
if ca_cert:
args.extend(['-ca_cert', ca_cert])
if socket_file:
args.extend(['-socket_file', socket_file])
if extra_args:
args.extend(extra_args)
sp = run_bg(args)
if cert:
wait_for_vars("vtgate", port, "SecureConnections")
return sp, port, secure_port
else:
wait_for_vars("vtgate", port)
return sp, port
def run_vtctl_vtctl(clargs, auto_log=False, expect_fail=False,
**kwargs):
args = environment.binary_args('vtctl') + ['-log_dir', environment.vtlogroot]
args.extend(environment.topo_server().flags())
args.extend(['-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol()])
args.extend(['-tablet_protocol', protocols_flavor().tabletconn_protocol()])
args.extend(['-vtgate_protocol', protocols_flavor().vtgate_protocol()])
if auto_log:
args.append('--stderrthreshold=%s' % get_log_level())
if isinstance(clargs, str):
cmd = ' '.join(args) + ' ' + clargs
else:
cmd = args + clargs
if expect_fail:
return run_fail(cmd, **kwargs)
return run(cmd, **kwargs)
def run_vtworker(clargs, log_level='', auto_log=False, expect_fail=False, **kwargs):
args = environment.binary_args('vtworker') + [
'-log_dir', environment.vtlogroot,
'-port', str(environment.reserve_ports(1))]
args.extend(environment.topo_server_flags())
args.extend(environment.tablet_manager_protocol_flags())
if auto_log:
if options.verbose == 2:
log_level='INFO'
elif options.verbose == 1:
log_level='WARNING'
else:
log_level='ERROR'
if log_level:
args.append('--stderrthreshold=%s' % log_level)
cmd = args + clargs
if expect_fail:
return run_fail(cmd, **kwargs)
return run(cmd, **kwargs)
def run_vtworker_client(args, rpc_port):
"""Runs vtworkerclient to execute a command on a remote vtworker.
Returns:
out - stdout of the vtworkerclient invocation
err - stderr of the vtworkerclient invocation
"""
if options.verbose == 2:
log_level = 'INFO'
elif options.verbose == 1:
log_level = 'WARNING'
else:
log_level = 'ERROR'
out, err = run(environment.binary_args('vtworkerclient') + [
'-vtworker_client_protocol',
protocols_flavor().vtworker_client_protocol(), '-server',
'localhost:%u' % rpc_port, '-stderrthreshold', log_level
] + args,
trap_output=True)
return out, err
def start(self):
args = environment.binary_args('vtctld') + [
'-debug',
'-templates', environment.vttop + '/go/cmd/vtctld/templates',
'-log_dir', environment.vtlogroot,
'-port', str(self.port),
] + \
environment.topo_server().flags() + \
protocols_flavor().tablet_manager_protocol_flags()
if protocols_flavor().vtctl_client_protocol() == "grpc":
args += [
'-grpc_port',
str(self.grpc_port), '-service_map', 'grpc-vtctl'
]
stderr_fd = open(os.path.join(environment.tmproot, "vtctld.stderr"),
"w")
self.proc = run_bg(args, stderr=stderr_fd)
# wait for the process to listen to RPC
timeout = 30
while True:
v = get_vars(self.port)
if v:
break
timeout = wait_step('waiting for vtctld to start',
timeout,
sleep_time=0.2)
# save the running instance so vtctl commands can be remote executed now
protocol = protocols_flavor().vtctl_client_protocol()
# temporary protocol override until python client support works
if protocol == "grpc":
protocol = "gorpc"
global vtctld, vtctld_connection
if not vtctld:
vtctld = self
vtctld_connection = vtctl_client.connect(
protocol, 'localhost:%u' % self.port, 30)
return self.proc
def mysqlctl(self, cmd, extra_my_cnf=None, with_ports=False, verbose=False):
all_extra_my_cnf = []
if environment.mysql_flavor == "GoogleMysql":
# we have to manually enable hierarchical replication to support group_id
all_extra_my_cnf.append(environment.vttop + "/config/mycnf/master_google.cnf")
if extra_my_cnf:
all_extra_my_cnf.append(extra_my_cnf)
extra_env = None
if all_extra_my_cnf:
extra_env = {
'EXTRA_MY_CNF': ":".join(all_extra_my_cnf),
}
args = environment.binary_args('mysqlctl') + [
'-log_dir', environment.vtlogroot,
'-tablet_uid', str(self.tablet_uid)]
if with_ports:
args.extend(['-port', str(self.port),
'-mysql_port', str(self.mysql_port)])
if verbose:
args.append('-alsologtostderr')
args.extend(cmd)
return utils.run_bg(args, extra_env=extra_env)
def _get_vtworker_cmd(clargs, auto_log=False):
"""Assembles the command that is needed to run a vtworker.
Args:
clargs: Command line arguments passed to vtworker.
auto_log: If true, set --stderrthreshold according to the test log level.
Returns:
cmd - list of cmd arguments, can be passed to any `run`-like functions
port - int with the port number that the vtworker is running with
rpc_port - int with the port number of the RPC interface
"""
port = environment.reserve_ports(1)
rpc_port = port
args = environment.binary_args('vtworker') + [
'-log_dir',
environment.vtlogroot,
'-port',
str(port),
'-executefetch_retry_time',
'1s',
'-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol(),
'-tablet_protocol',
protocols_flavor().tabletconn_protocol(),
]
args.extend(environment.topo_server().flags())
if protocols_flavor().service_map():
args.extend(
['-service_map', ','.join(protocols_flavor().service_map())])
if protocols_flavor().vtworker_client_protocol() == 'grpc':
rpc_port = environment.reserve_ports(1)
args.extend(['-grpc_port', str(rpc_port)])
if auto_log:
args.append('--stderrthreshold=%s' % get_log_level())
cmd = args + clargs
return cmd, port, rpc_port
