def run_vtctl(clargs, log_level='', auto_log=False, expect_fail=False, **kwargs):
args = [environment.binary_path('vtctl'), '-log_dir', environment.vtlogroot]
args.extend(environment.topo_server_flags())
args.extend(environment.tablet_manager_protocol_flags())
args.extend(environment.tabletconn_protocol_flags())
if auto_log:
if options.verbose == 2:
log_level='INFO'
elif options.verbose == 1:
log_level='WARNING'
else:
log_level='ERROR'
if log_level:
args.append('--stderrthreshold=%s' % log_level)
if isinstance(clargs, str):
cmd = " ".join(args) + ' ' + clargs
else:
cmd = args + clargs
if expect_fail:
return run_fail(cmd, **kwargs)
return run(cmd, **kwargs)
def vtclient2(uid, path, query, bindvars=None, user=None, password=None, driver=None,
verbose=False, raise_on_error=True):
if (user is None) != (password is None):
raise TypeError("you should provide either both or none of user and password")
# for ZK paths to not have // in the path, that confuses things
if path.startswith('/'):
path = path[1:]
server = "localhost:%u/%s" % (uid, path)
cmdline = [environment.binary_path('vtclient2'), '-server', server]
cmdline += environment.topo_server_flags()
cmdline += environment.tabletconn_protocol_flags()
if user is not None:
cmdline.extend(['-tablet-bson-username', user,
'-tablet-bson-password', password])
if bindvars:
cmdline.extend(['-bindvars', bindvars])
if driver:
cmdline.extend(['-driver', driver])
if verbose:
cmdline.extend(['-alsologtostderr', '-verbose'])
cmdline.append(query)
return run(cmdline, raise_on_error=raise_on_error, trap_output=True)
def run_vtctl(clargs, log_level='', auto_log=False, expect_fail=False, **kwargs):
args = environment.binary_args('vtctl') + ['-log_dir', environment.vtlogroot]
args.extend(environment.topo_server_flags())
args.extend(environment.tablet_manager_protocol_flags())
args.extend(environment.tabletconn_protocol_flags())
if auto_log:
if options.verbose == 2:
log_level='INFO'
elif options.verbose == 1:
log_level='WARNING'
else:
log_level='ERROR'
if log_level:
args.append('--stderrthreshold=%s' % log_level)
if isinstance(clargs, str):
cmd = " ".join(args) + ' ' + clargs
else:
cmd = args + clargs
if expect_fail:
return run_fail(cmd, **kwargs)
return run(cmd, **kwargs)
def vtclient2(uid, path, query, bindvars=None, user=None, password=None, driver=None,
verbose=False, raise_on_error=True):
if (user is None) != (password is None):
raise TypeError("you should provide either both or none of user and password")
# for ZK paths to not have // in the path, that confuses things
if path.startswith('/'):
path = path[1:]
server = "localhost:%u/%s" % (uid, path)
cmdline = environment.binary_args('vtclient2') + ['-server', server]
cmdline += environment.topo_server_flags()
cmdline += environment.tabletconn_protocol_flags()
if user is not None:
cmdline.extend(['-tablet-bson-username', user,
'-tablet-bson-password', password])
if bindvars:
cmdline.extend(['-bindvars', bindvars])
if driver:
cmdline.extend(['-driver', driver])
if verbose:
cmdline.extend(['-alsologtostderr', '-verbose'])
cmdline.append(query)
return run(cmdline, raise_on_error=raise_on_error, trap_output=True)
def vtgate_start(vtport=None, cell='test_nj', retry_delay=1, retry_count=1, topo_impl=None, tablet_bson_encrypted=False, cache_ttl='1s', auth=False, timeout="5s", cert=None, key=None, ca_cert=None):
port = vtport or environment.reserve_ports(1)
secure_port = None
args = [environment.binary_path('vtgate'),
'-port', str(port),
'-cell', cell,
'-retry-delay', '%ss' % (str(retry_delay)),
'-retry-count', str(retry_count),
'-log_dir', environment.vtlogroot,
'-srv_topo_cache_ttl', cache_ttl,
'-timeout', timeout,
] + environment.tabletconn_protocol_flags()
if topo_impl:
args.extend(['-topo_implementation', topo_impl])
else:
args.extend(environment.topo_server_flags())
if tablet_bson_encrypted:
args.append('-tablet-bson-encrypted')
if auth:
args.extend(['-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json')])
if cert:
secure_port = environment.reserve_ports(1)
args.extend(['-secure-port', '%s' % secure_port,
'-cert', cert,
'-key', key])
if ca_cert:
args.extend(['-ca-cert', ca_cert])
sp = run_bg(args)
if cert:
wait_for_vars("vtgate", port, "SecureConnections")
return sp, port, secure_port
else:
wait_for_vars("vtgate", port)
return sp, port
def test_actions_and_timeouts(self):
# Start up a master mysql and vttablet
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
tablet_62344.init_tablet('master', 'test_keyspace', '0')
utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
utils.validate_topology()
srvShard = utils.run_vtctl_json(['GetSrvShard', 'test_nj',
'test_keyspace/0'])
self.assertEqual(srvShard['MasterCell'], 'test_nj')
tablet_62344.create_db('vt_test_keyspace')
tablet_62344.start_vttablet()
utils.run_vtctl(['RpcPing', tablet_62344.tablet_alias])
# schedule long action in the background, sleep a little bit to make sure
# it started to run
args = (environment.binary_args('vtctl') +
environment.topo_server_flags() +
environment.tablet_manager_protocol_flags() +
environment.tabletconn_protocol_flags() +
['-log_dir', environment.vtlogroot,
'Sleep', tablet_62344.tablet_alias, '10s'])
bg = utils.run_bg(args)
time.sleep(3)
# try a frontend RpcPing that should timeout as the tablet is busy
# running the other one
stdout, stderr = utils.run_vtctl(['-wait-time', '3s',
'RpcPing', tablet_62344.tablet_alias],
expect_fail=True)
if 'Timeout waiting for' not in stderr:
self.fail("didn't find the right error strings in failed RpcPing: " +
stderr)
# wait for the background vtctl
bg.wait()
if environment.topo_server_implementation == 'zookeeper':
# extra small test: we ran for a while, get the states we were in,
# make sure they're accounted for properly
# first the query engine States
v = utils.get_vars(tablet_62344.port)
logging.debug("vars: %s" % str(v))
# then the Zookeeper connections
if v['ZkMetaConn']['test_nj']['Current'] != 'Connected':
self.fail('invalid zk test_nj state: %s' %
v['ZkMetaConn']['test_nj']['Current'])
if v['ZkMetaConn']['global']['Current'] != 'Connected':
self.fail('invalid zk global state: %s' %
v['ZkMetaConn']['global']['Current'])
if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9:
self.fail('not enough time in Connected state: %u',
v['ZkMetaConn']['test_nj']['DurationConnected'])
if v['TabletType'] != 'master':
self.fail('TabletType not exported correctly')
tablet_62344.kill_vttablet()
def test_actions_and_timeouts(self):
# Start up a master mysql and vttablet
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
tablet_62344.init_tablet('master', 'test_keyspace', '0')
utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
utils.validate_topology()
srvShard = utils.run_vtctl_json(
['GetSrvShard', 'test_nj', 'test_keyspace/0'])
self.assertEqual(srvShard['MasterCell'], 'test_nj')
tablet_62344.create_db('vt_test_keyspace')
tablet_62344.start_vttablet()
utils.run_vtctl(['Ping', tablet_62344.tablet_alias])
# schedule long action in the background, sleep a little bit to make sure
# it started to run
args = (environment.binary_args('vtctl') +
environment.topo_server_flags() +
environment.tablet_manager_protocol_flags() +
environment.tabletconn_protocol_flags() + [
'-log_dir', environment.vtlogroot, 'Sleep',
tablet_62344.tablet_alias, '10s'
])
bg = utils.run_bg(args)
time.sleep(3)
# try a frontend RefreshState that should timeout as the tablet is busy
# running the other one
stdout, stderr = utils.run_vtctl(
['-wait-time', '3s', 'RefreshState', tablet_62344.tablet_alias],
expect_fail=True)
self.assertIn(environment.rpc_timeout_message, stderr)
# wait for the background vtctl
bg.wait()
if environment.topo_server_implementation == 'zookeeper':
# extra small test: we ran for a while, get the states we were in,
# make sure they're accounted for properly
# first the query engine States
v = utils.get_vars(tablet_62344.port)
logging.debug("vars: %s" % str(v))
# then the Zookeeper connections
if v['ZkMetaConn']['test_nj']['Current'] != 'Connected':
self.fail('invalid zk test_nj state: %s' %
v['ZkMetaConn']['test_nj']['Current'])
if v['ZkMetaConn']['global']['Current'] != 'Connected':
self.fail('invalid zk global state: %s' %
v['ZkMetaConn']['global']['Current'])
if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9:
self.fail('not enough time in Connected state: %u',
v['ZkMetaConn']['test_nj']['DurationConnected'])
if v['TabletType'] != 'master':
self.fail('TabletType not exported correctly')
tablet_62344.kill_vttablet()
def vtgate_start(vtport=None,
cell='test_nj',
retry_delay=1,
retry_count=1,
topo_impl=None,
tablet_bson_encrypted=False,
cache_ttl='1s',
auth=False,
timeout="5s",
cert=None,
key=None,
ca_cert=None):
port = vtport or environment.reserve_ports(1)
secure_port = None
args = [
environment.binary_path('vtgate'),
'-port',
str(port),
'-cell',
cell,
'-retry-delay',
'%ss' % (str(retry_delay)),
'-retry-count',
str(retry_count),
'-log_dir',
environment.vtlogroot,
'-srv_topo_cache_ttl',
cache_ttl,
'-timeout',
timeout,
] + environment.tabletconn_protocol_flags()
if topo_impl:
args.extend(['-topo_implementation', topo_impl])
else:
args.extend(environment.topo_server_flags())
if tablet_bson_encrypted:
args.append('-tablet-bson-encrypted')
if auth:
args.extend([
'-auth-credentials',
os.path.join(environment.vttop, 'test', 'test_data',
'authcredentials_test.json')
])
if cert:
secure_port = environment.reserve_ports(1)
args.extend(
['-secure-port',
'%s' % secure_port, '-cert', cert, '-key', key])
if ca_cert:
args.extend(['-ca-cert', ca_cert])
sp = run_bg(args)
if cert:
wait_for_vars("vtgate", port, "SecureConnections")
return sp, port, secure_port
else:
wait_for_vars("vtgate", port)
return sp, port
def vtgate_start(cell='test_nj', retry_delay=1, retry_count=1, topo_impl=None, tablet_bson_encrypted=False):
port = environment.reserve_ports(1)
args = [environment.binary_path('vtgate'),
'-port', str(port),
'-cell', cell,
'-retry-delay', '%ss' % (str(retry_delay)),
'-retry-count', str(retry_count),
'-log_dir', environment.vtlogroot,
] + environment.tabletconn_protocol_flags()
if topo_impl:
args.extend(['-topo_implementation', topo_impl])
else:
args.extend(environment.topo_server_flags())
if tablet_bson_encrypted:
args.append('-tablet-bson-encrypted')
sp = run_bg(args)
wait_for_vars("vtgate", port)
return sp, port
