def run_vtctl(clargs, log_level='', auto_log=False, expect_fail=False, **kwargs): args = [environment.binary_path('vtctl'), '-log_dir', environment.vtlogroot] args.extend(environment.topo_server_flags()) args.extend(environment.tablet_manager_protocol_flags()) args.extend(environment.tabletconn_protocol_flags()) if auto_log: if options.verbose == 2: log_level='INFO' elif options.verbose == 1: log_level='WARNING' else: log_level='ERROR' if log_level: args.append('--stderrthreshold=%s' % log_level) if isinstance(clargs, str): cmd = " ".join(args) + ' ' + clargs else: cmd = args + clargs if expect_fail: return run_fail(cmd, **kwargs) return run(cmd, **kwargs)
def vtclient2(uid, path, query, bindvars=None, user=None, password=None, driver=None, verbose=False, raise_on_error=True): if (user is None) != (password is None): raise TypeError("you should provide either both or none of user and password") # for ZK paths to not have // in the path, that confuses things if path.startswith('/'): path = path[1:] server = "localhost:%u/%s" % (uid, path) cmdline = [environment.binary_path('vtclient2'), '-server', server] cmdline += environment.topo_server_flags() cmdline += environment.tabletconn_protocol_flags() if user is not None: cmdline.extend(['-tablet-bson-username', user, '-tablet-bson-password', password]) if bindvars: cmdline.extend(['-bindvars', bindvars]) if driver: cmdline.extend(['-driver', driver]) if verbose: cmdline.extend(['-alsologtostderr', '-verbose']) cmdline.append(query) return run(cmdline, raise_on_error=raise_on_error, trap_output=True)
def run_vtctl(clargs, log_level='', auto_log=False, expect_fail=False, **kwargs): args = environment.binary_args('vtctl') + ['-log_dir', environment.vtlogroot] args.extend(environment.topo_server_flags()) args.extend(environment.tablet_manager_protocol_flags()) args.extend(environment.tabletconn_protocol_flags()) if auto_log: if options.verbose == 2: log_level='INFO' elif options.verbose == 1: log_level='WARNING' else: log_level='ERROR' if log_level: args.append('--stderrthreshold=%s' % log_level) if isinstance(clargs, str): cmd = " ".join(args) + ' ' + clargs else: cmd = args + clargs if expect_fail: return run_fail(cmd, **kwargs) return run(cmd, **kwargs)
def vtclient2(uid, path, query, bindvars=None, user=None, password=None, driver=None, verbose=False, raise_on_error=True): if (user is None) != (password is None): raise TypeError("you should provide either both or none of user and password") # for ZK paths to not have // in the path, that confuses things if path.startswith('/'): path = path[1:] server = "localhost:%u/%s" % (uid, path) cmdline = environment.binary_args('vtclient2') + ['-server', server] cmdline += environment.topo_server_flags() cmdline += environment.tabletconn_protocol_flags() if user is not None: cmdline.extend(['-tablet-bson-username', user, '-tablet-bson-password', password]) if bindvars: cmdline.extend(['-bindvars', bindvars]) if driver: cmdline.extend(['-driver', driver]) if verbose: cmdline.extend(['-alsologtostderr', '-verbose']) cmdline.append(query) return run(cmdline, raise_on_error=raise_on_error, trap_output=True)
def vtgate_start(vtport=None, cell='test_nj', retry_delay=1, retry_count=1, topo_impl=None, tablet_bson_encrypted=False, cache_ttl='1s', auth=False, timeout="5s", cert=None, key=None, ca_cert=None): port = vtport or environment.reserve_ports(1) secure_port = None args = [environment.binary_path('vtgate'), '-port', str(port), '-cell', cell, '-retry-delay', '%ss' % (str(retry_delay)), '-retry-count', str(retry_count), '-log_dir', environment.vtlogroot, '-srv_topo_cache_ttl', cache_ttl, '-timeout', timeout, ] + environment.tabletconn_protocol_flags() if topo_impl: args.extend(['-topo_implementation', topo_impl]) else: args.extend(environment.topo_server_flags()) if tablet_bson_encrypted: args.append('-tablet-bson-encrypted') if auth: args.extend(['-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json')]) if cert: secure_port = environment.reserve_ports(1) args.extend(['-secure-port', '%s' % secure_port, '-cert', cert, '-key', key]) if ca_cert: args.extend(['-ca-cert', ca_cert]) sp = run_bg(args) if cert: wait_for_vars("vtgate", port, "SecureConnections") return sp, port, secure_port else: wait_for_vars("vtgate", port) return sp, port
def test_actions_and_timeouts(self): # Start up a master mysql and vttablet utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) tablet_62344.init_tablet('master', 'test_keyspace', '0') utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0']) utils.validate_topology() srvShard = utils.run_vtctl_json(['GetSrvShard', 'test_nj', 'test_keyspace/0']) self.assertEqual(srvShard['MasterCell'], 'test_nj') tablet_62344.create_db('vt_test_keyspace') tablet_62344.start_vttablet() utils.run_vtctl(['RpcPing', tablet_62344.tablet_alias]) # schedule long action in the background, sleep a little bit to make sure # it started to run args = (environment.binary_args('vtctl') + environment.topo_server_flags() + environment.tablet_manager_protocol_flags() + environment.tabletconn_protocol_flags() + ['-log_dir', environment.vtlogroot, 'Sleep', tablet_62344.tablet_alias, '10s']) bg = utils.run_bg(args) time.sleep(3) # try a frontend RpcPing that should timeout as the tablet is busy # running the other one stdout, stderr = utils.run_vtctl(['-wait-time', '3s', 'RpcPing', tablet_62344.tablet_alias], expect_fail=True) if 'Timeout waiting for' not in stderr: self.fail("didn't find the right error strings in failed RpcPing: " + stderr) # wait for the background vtctl bg.wait() if environment.topo_server_implementation == 'zookeeper': # extra small test: we ran for a while, get the states we were in, # make sure they're accounted for properly # first the query engine States v = utils.get_vars(tablet_62344.port) logging.debug("vars: %s" % str(v)) # then the Zookeeper connections if v['ZkMetaConn']['test_nj']['Current'] != 'Connected': self.fail('invalid zk test_nj state: %s' % v['ZkMetaConn']['test_nj']['Current']) if v['ZkMetaConn']['global']['Current'] != 'Connected': self.fail('invalid zk global state: %s' % v['ZkMetaConn']['global']['Current']) if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9: self.fail('not enough time in Connected state: %u', v['ZkMetaConn']['test_nj']['DurationConnected']) if v['TabletType'] != 'master': self.fail('TabletType not exported correctly') tablet_62344.kill_vttablet()
def test_actions_and_timeouts(self): # Start up a master mysql and vttablet utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) tablet_62344.init_tablet('master', 'test_keyspace', '0') utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0']) utils.validate_topology() srvShard = utils.run_vtctl_json( ['GetSrvShard', 'test_nj', 'test_keyspace/0']) self.assertEqual(srvShard['MasterCell'], 'test_nj') tablet_62344.create_db('vt_test_keyspace') tablet_62344.start_vttablet() utils.run_vtctl(['Ping', tablet_62344.tablet_alias]) # schedule long action in the background, sleep a little bit to make sure # it started to run args = (environment.binary_args('vtctl') + environment.topo_server_flags() + environment.tablet_manager_protocol_flags() + environment.tabletconn_protocol_flags() + [ '-log_dir', environment.vtlogroot, 'Sleep', tablet_62344.tablet_alias, '10s' ]) bg = utils.run_bg(args) time.sleep(3) # try a frontend RefreshState that should timeout as the tablet is busy # running the other one stdout, stderr = utils.run_vtctl( ['-wait-time', '3s', 'RefreshState', tablet_62344.tablet_alias], expect_fail=True) self.assertIn(environment.rpc_timeout_message, stderr) # wait for the background vtctl bg.wait() if environment.topo_server_implementation == 'zookeeper': # extra small test: we ran for a while, get the states we were in, # make sure they're accounted for properly # first the query engine States v = utils.get_vars(tablet_62344.port) logging.debug("vars: %s" % str(v)) # then the Zookeeper connections if v['ZkMetaConn']['test_nj']['Current'] != 'Connected': self.fail('invalid zk test_nj state: %s' % v['ZkMetaConn']['test_nj']['Current']) if v['ZkMetaConn']['global']['Current'] != 'Connected': self.fail('invalid zk global state: %s' % v['ZkMetaConn']['global']['Current']) if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9: self.fail('not enough time in Connected state: %u', v['ZkMetaConn']['test_nj']['DurationConnected']) if v['TabletType'] != 'master': self.fail('TabletType not exported correctly') tablet_62344.kill_vttablet()
def vtgate_start(vtport=None, cell='test_nj', retry_delay=1, retry_count=1, topo_impl=None, tablet_bson_encrypted=False, cache_ttl='1s', auth=False, timeout="5s", cert=None, key=None, ca_cert=None): port = vtport or environment.reserve_ports(1) secure_port = None args = [ environment.binary_path('vtgate'), '-port', str(port), '-cell', cell, '-retry-delay', '%ss' % (str(retry_delay)), '-retry-count', str(retry_count), '-log_dir', environment.vtlogroot, '-srv_topo_cache_ttl', cache_ttl, '-timeout', timeout, ] + environment.tabletconn_protocol_flags() if topo_impl: args.extend(['-topo_implementation', topo_impl]) else: args.extend(environment.topo_server_flags()) if tablet_bson_encrypted: args.append('-tablet-bson-encrypted') if auth: args.extend([ '-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json') ]) if cert: secure_port = environment.reserve_ports(1) args.extend( ['-secure-port', '%s' % secure_port, '-cert', cert, '-key', key]) if ca_cert: args.extend(['-ca-cert', ca_cert]) sp = run_bg(args) if cert: wait_for_vars("vtgate", port, "SecureConnections") return sp, port, secure_port else: wait_for_vars("vtgate", port) return sp, port
def vtgate_start(cell='test_nj', retry_delay=1, retry_count=1, topo_impl=None, tablet_bson_encrypted=False): port = environment.reserve_ports(1) args = [environment.binary_path('vtgate'), '-port', str(port), '-cell', cell, '-retry-delay', '%ss' % (str(retry_delay)), '-retry-count', str(retry_count), '-log_dir', environment.vtlogroot, ] + environment.tabletconn_protocol_flags() if topo_impl: args.extend(['-topo_implementation', topo_impl]) else: args.extend(environment.topo_server_flags()) if tablet_bson_encrypted: args.append('-tablet-bson-encrypted') sp = run_bg(args) wait_for_vars("vtgate", port) return sp, port