def run_vtctl_vtctl(clargs, auto_log=False, expect_fail=False, **kwargs): args = environment.binary_args('vtctl') + [ '-log_dir', environment.vtlogroot, '-enable_queries', ] args.extend(environment.topo_server().flags()) args.extend(['-tablet_manager_protocol', protocols_flavor().tablet_manager_protocol()]) args.extend(['-tablet_protocol', protocols_flavor().tabletconn_protocol()]) args.extend(['-throttler_client_protocol', protocols_flavor().throttler_client_protocol()]) args.extend(['-vtgate_protocol', protocols_flavor().vtgate_protocol()]) # TODO(b/26388813): Remove the next two lines once vtctl WaitForDrain is # integrated in the vtctl MigrateServed* commands. args.extend(['--wait_for_drain_sleep_rdonly', '0s']) args.extend(['--wait_for_drain_sleep_replica', '0s']) if auto_log: args.append('--stderrthreshold=%s' % get_log_level()) if isinstance(clargs, str): cmd = ' '.join(args) + ' ' + clargs else: cmd = args + clargs if expect_fail: return run_fail(cmd, **kwargs) return run(cmd, **kwargs)
def run_vtctl_vtctl(clargs, auto_log=False, expect_fail=False, **kwargs): args = environment.binary_args('vtctl') + [ '-log_dir', environment.vtlogroot, '-enable_queries', ] args.extend(environment.topo_server().flags()) args.extend(['-tablet_manager_protocol', protocols_flavor().tablet_manager_protocol()]) args.extend(['-tablet_protocol', protocols_flavor().tabletconn_protocol()]) args.extend(['-throttler_client_protocol', protocols_flavor().throttler_client_protocol()]) args.extend(['-vtgate_protocol', protocols_flavor().vtgate_protocol()]) if auto_log: args.append('--stderrthreshold=%s' % get_log_level()) if isinstance(clargs, str): cmd = ' '.join(args) + ' ' + clargs else: cmd = args + clargs if expect_fail: return run_fail(cmd, **kwargs) return run(cmd, **kwargs)
def rpc_endpoint(self, python=False): """RPC endpoint to vtctld. The RPC endpoint may differ from the webinterface URL e.g. because gRPC requires a dedicated port. Args: python: boolean, True iff this is for access with Python (as opposed to Go). Returns: protocol - string e.g. 'grpc' endpoint - string e.g. 'localhost:15001' """ if python: protocol = protocols_flavor().vtctl_python_client_protocol() else: protocol = protocols_flavor().vtctl_client_protocol() rpc_port = self.port if protocol == 'grpc': # import the grpc vtctl client implementation, change the port if python: from vtctl import grpc_vtctl_client # pylint: disable=g-import-not-at-top,unused-variable rpc_port = self.grpc_port return (protocol, '%s:%d' % (socket.getfqdn(), rpc_port))
def _get_vtworker_cmd(clargs, auto_log=False): """Assembles the command that is needed to run a vtworker. Returns: cmd - list of cmd arguments, can be passed to any `run`-like functions port - int with the port number that the vtworker is running with rpc_port - int with the port number of the RPC interface """ port = environment.reserve_ports(1) rpc_port = port args = environment.binary_args('vtworker') + [ '-log_dir', environment.vtlogroot, '-min_healthy_rdonly_endpoints', '1', '-port', str(port), '-resolve_ttl', '2s', '-executefetch_retry_time', '1s', ] args.extend(environment.topo_server().flags()) args.extend(['-tablet_manager_protocol', protocols_flavor().tablet_manager_protocol()]) if protocols_flavor().service_map(): args.extend(['-service_map', ",".join(protocols_flavor().service_map())]) if protocols_flavor().vtworker_client_protocol() == 'grpc': rpc_port = environment.reserve_ports(1) args.extend(['-grpc_port', str(rpc_port)]) if auto_log: args.append('--stderrthreshold=%s' % get_log_level()) cmd = args + clargs return cmd, port, rpc_port
def test_actions_and_timeouts(self): # Start up a master mysql and vttablet utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) tablet_62344.init_tablet('master', 'test_keyspace', '0') utils.validate_topology() tablet_62344.create_db('vt_test_keyspace') tablet_62344.start_vttablet() utils.run_vtctl(['Ping', tablet_62344.tablet_alias]) # schedule long action in the background, sleep a little bit to make sure # it started to run args = (environment.binary_args('vtctl') + environment.topo_server().flags() + ['-tablet_manager_protocol', protocols_flavor().tablet_manager_protocol(), '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-log_dir', environment.vtlogroot, 'Sleep', tablet_62344.tablet_alias, '10s']) bg = utils.run_bg(args) time.sleep(3) # try a frontend RefreshState that should timeout as the tablet is busy # running the other one _, stderr = utils.run_vtctl( ['-wait-time', '3s', 'RefreshState', tablet_62344.tablet_alias], expect_fail=True) self.assertIn(protocols_flavor().rpc_timeout_message(), stderr) # wait for the background vtctl bg.wait() tablet_62344.kill_vttablet()
def run_vtctl_vtctl(clargs, log_level='', auto_log=False, expect_fail=False, **kwargs): args = environment.binary_args('vtctl') + ['-log_dir', environment.vtlogroot] args.extend(environment.topo_server().flags()) args.extend(protocols_flavor().tablet_manager_protocol_flags()) args.extend(protocols_flavor().tabletconn_protocol_flags()) if auto_log: if options.verbose == 2: log_level='INFO' elif options.verbose == 1: log_level='WARNING' else: log_level='ERROR' if log_level: args.append('--stderrthreshold=%s' % log_level) if isinstance(clargs, str): cmd = " ".join(args) + ' ' + clargs else: cmd = args + clargs if expect_fail: return run_fail(cmd, **kwargs) return run(cmd, **kwargs)
def start(self): args = environment.binary_args('vtctld') + [ '-debug', '-templates', environment.vttop + '/go/cmd/vtctld/templates', '-log_dir', environment.vtlogroot, '-port', str(self.port), ] + \ environment.topo_server().flags() + \ protocols_flavor().tablet_manager_protocol_flags() stderr_fd = open(os.path.join(environment.tmproot, "vtctld.stderr"), "w") self.proc = run_bg(args, stderr=stderr_fd) # wait for the process to listen to RPC timeout = 30 while True: v = get_vars(self.port) if v: break timeout = wait_step('waiting for vtctld to start', timeout, sleep_time=0.2) # save the running instance so vtctl commands can be remote executed now global vtctld, vtctld_connection if not vtctld: vtctld = self vtctld_connection = vtctl_client.connect( protocols_flavor().vtctl_client_protocol(), 'localhost:%u' % self.port, 30) return self.proc
def setUpModule(): global vtgateclienttest_process global vtgateclienttest_port global vtgateclienttest_grpc_port try: environment.topo_server().setup() vtgateclienttest_port = environment.reserve_ports(1) args = environment.binary_args('vtgateclienttest') + [ '-log_dir', environment.vtlogroot, '-port', str(vtgateclienttest_port), ] if protocols_flavor().vtgate_python_protocol() == 'grpc': vtgateclienttest_grpc_port = environment.reserve_ports(1) args.extend(['-grpc_port', str(vtgateclienttest_grpc_port)]) if protocols_flavor().service_map(): args.extend(['-service_map', ','.join(protocols_flavor().service_map())]) vtgateclienttest_process = utils.run_bg(args) utils.wait_for_vars('vtgateclienttest', vtgateclienttest_port) except: tearDownModule() raise
def test_actions_and_timeouts(self): # Start up a master mysql and vttablet utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) tablet_62344.init_tablet('master', 'test_keyspace', '0') utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0']) utils.validate_topology() srvShard = utils.run_vtctl_json(['GetSrvShard', 'test_nj', 'test_keyspace/0']) self.assertEqual(srvShard['MasterCell'], 'test_nj') tablet_62344.create_db('vt_test_keyspace') tablet_62344.start_vttablet() utils.run_vtctl(['Ping', tablet_62344.tablet_alias]) # schedule long action in the background, sleep a little bit to make sure # it started to run args = (environment.binary_args('vtctl') + environment.topo_server().flags() + ['-tablet_manager_protocol', protocols_flavor().tablet_manager_protocol(), '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-log_dir', environment.vtlogroot, 'Sleep', tablet_62344.tablet_alias, '10s']) bg = utils.run_bg(args) time.sleep(3) # try a frontend RefreshState that should timeout as the tablet is busy # running the other one stdout, stderr = utils.run_vtctl(['-wait-time', '3s', 'RefreshState', tablet_62344.tablet_alias], expect_fail=True) self.assertIn(protocols_flavor().rpc_timeout_message(), stderr) # wait for the background vtctl bg.wait() if environment.topo_server().flavor() == 'zookeeper': # extra small test: we ran for a while, get the states we were in, # make sure they're accounted for properly # first the query engine States v = utils.get_vars(tablet_62344.port) logging.debug("vars: %s" % str(v)) # then the Zookeeper connections if v['ZkMetaConn']['test_nj']['Current'] != 'Connected': self.fail('invalid zk test_nj state: %s' % v['ZkMetaConn']['test_nj']['Current']) if v['ZkMetaConn']['global']['Current'] != 'Connected': self.fail('invalid zk global state: %s' % v['ZkMetaConn']['global']['Current']) if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9: self.fail('not enough time in Connected state: %u', v['ZkMetaConn']['test_nj']['DurationConnected']) if v['TabletType'] != 'master': self.fail('TabletType not exported correctly') tablet_62344.kill_vttablet()
def start( self, cell="test_nj", retry_delay=1, retry_count=2, topo_impl=None, cache_ttl="1s", timeout_total="2s", timeout_per_conn="1s", extra_args=None, tablets=None, ): """Start vtgate. Saves it into the global vtgate variable if not set yet.""" args = environment.binary_args("vtgate") + [ "-port", str(self.port), "-cell", cell, "-retry-delay", "%ss" % (str(retry_delay)), "-retry-count", str(retry_count), "-log_dir", environment.vtlogroot, "-srv_topo_cache_ttl", cache_ttl, "-conn-timeout-total", timeout_total, "-conn-timeout-per-conn", timeout_per_conn, "-bsonrpc_timeout", "5s", "-tablet_protocol", protocols_flavor().tabletconn_protocol(), "-gateway_implementation", vtgate_gateway_flavor().flavor(), "-tablet_types_to_wait", "MASTER,REPLICA", ] args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets)) if protocols_flavor().vtgate_protocol() == "grpc": args.extend(["-grpc_port", str(self.grpc_port)]) if protocols_flavor().service_map(): args.extend(["-service_map", ",".join(protocols_flavor().service_map())]) if topo_impl: args.extend(["-topo_implementation", topo_impl]) else: args.extend(environment.topo_server().flags()) if extra_args: args.extend(extra_args) self.proc = run_bg(args) wait_for_vars("vtgate", self.port) global vtgate if not vtgate: vtgate = self
def start( self, cell="test_nj", retry_delay=1, retry_count=2, topo_impl=None, cache_ttl="1s", timeout_total="4s", timeout_per_conn="2s", extra_args=None, ): """Starts the process for this vtgate instance. If no other instance has been started, saves it into the global vtgate variable. """ args = environment.binary_args("vtgate") + [ "-port", str(self.port), "-cell", cell, "-retry-delay", "%ss" % (str(retry_delay)), "-retry-count", str(retry_count), "-log_dir", environment.vtlogroot, "-srv_topo_cache_ttl", cache_ttl, "-conn-timeout-total", timeout_total, "-conn-timeout-per-conn", timeout_per_conn, "-bsonrpc_timeout", "5s", "-tablet_protocol", protocols_flavor().tabletconn_protocol(), ] if protocols_flavor().vtgate_protocol() == "grpc": args.extend(["-grpc_port", str(self.grpc_port)]) if protocols_flavor().service_map(): args.extend(["-service_map", ",".join(protocols_flavor().service_map())]) if topo_impl: args.extend(["-topo_implementation", topo_impl]) else: args.extend(environment.topo_server().flags()) if extra_args: args.extend(extra_args) self.proc = run_bg(args) if self.secure_port: wait_for_vars("vtgate", self.port, "SecureConnections") else: wait_for_vars("vtgate", self.port) global vtgate if not vtgate: vtgate = self
def test_actions_and_timeouts(self): # Start up a master mysql and vttablet utils.run_vtctl(["CreateKeyspace", "test_keyspace"]) tablet_62344.init_tablet("master", "test_keyspace", "0") utils.run_vtctl(["RebuildShardGraph", "test_keyspace/0"]) utils.validate_topology() self._check_srv_shard() tablet_62344.create_db("vt_test_keyspace") tablet_62344.start_vttablet() utils.run_vtctl(["Ping", tablet_62344.tablet_alias]) # schedule long action in the background, sleep a little bit to make sure # it started to run args = ( environment.binary_args("vtctl") + environment.topo_server().flags() + [ "-tablet_manager_protocol", protocols_flavor().tablet_manager_protocol(), "-tablet_protocol", protocols_flavor().tabletconn_protocol(), "-log_dir", environment.vtlogroot, "Sleep", tablet_62344.tablet_alias, "10s", ] ) bg = utils.run_bg(args) time.sleep(3) # try a frontend RefreshState that should timeout as the tablet is busy # running the other one _, stderr = utils.run_vtctl(["-wait-time", "3s", "RefreshState", tablet_62344.tablet_alias], expect_fail=True) self.assertIn(protocols_flavor().rpc_timeout_message(), stderr) # wait for the background vtctl bg.wait() if environment.topo_server().flavor() == "zookeeper": # extra small test: we ran for a while, get the states we were in, # make sure they're accounted for properly # first the query engine States v = utils.get_vars(tablet_62344.port) logging.debug("vars: %s", v) # then the Zookeeper connections if v["ZkCachedConn"]["test_nj"] != "Connected": self.fail("invalid zk test_nj state: %s" % v["ZkCachedConn"]["test_nj"]) if v["ZkCachedConn"]["global"] != "Connected": self.fail("invalid zk global state: %s" % v["ZkCachedConn"]["global"]) if v["TabletType"] != "master": self.fail("TabletType not exported correctly") tablet_62344.kill_vttablet()
def start(self, cell='test_nj', retry_delay=1, retry_count=2, topo_impl=None, cache_ttl='1s', timeout_total='2s', timeout_per_conn='1s', extra_args=None, tablets=None): """Start vtgate. Saves it into the global vtgate variable if not set yet.""" args = environment.binary_args('vtgate') + [ '-port', str(self.port), '-cell', cell, '-retry-delay', '%ss' % (str(retry_delay)), '-retry-count', str(retry_count), '-log_dir', environment.vtlogroot, '-srv_topo_cache_ttl', cache_ttl, '-conn-timeout-total', timeout_total, '-conn-timeout-per-conn', timeout_per_conn, '-bsonrpc_timeout', '5s', '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-gateway_implementation', vtgate_gateway_flavor().flavor(), '-tablet_types_to_wait', 'MASTER,REPLICA', ] args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets)) if protocols_flavor().vtgate_protocol() == 'grpc': args.extend(['-grpc_port', str(self.grpc_port)]) if protocols_flavor().service_map(): args.extend( ['-service_map', ','.join(protocols_flavor().service_map())]) if topo_impl: args.extend(['-topo_implementation', topo_impl]) else: args.extend(environment.topo_server().flags()) if extra_args: args.extend(extra_args) self.proc = run_bg(args) wait_for_vars('vtgate', self.port) global vtgate if not vtgate: vtgate = self
def rpc_endpoint(self, python=False): """Returns the protocol and endpoint to use for RPCs.""" if python: protocol = protocols_flavor().vtgate_python_protocol() else: protocol = protocols_flavor().vtgate_protocol() if protocol == 'grpc': return protocol, 'localhost:%d' % self.grpc_port return protocol, self.addr()
def test_actions_and_timeouts(self): # Start up a master mysql and vttablet utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) tablet_62344.init_tablet('master', 'test_keyspace', '0') utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0']) utils.validate_topology() self._check_srv_shard() tablet_62344.create_db('vt_test_keyspace') tablet_62344.start_vttablet() utils.run_vtctl(['Ping', tablet_62344.tablet_alias]) # schedule long action in the background, sleep a little bit to make sure # it started to run args = (environment.binary_args('vtctl') + environment.topo_server().flags() + [ '-tablet_manager_protocol', protocols_flavor().tablet_manager_protocol(), '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-log_dir', environment.vtlogroot, 'Sleep', tablet_62344.tablet_alias, '10s' ]) bg = utils.run_bg(args) time.sleep(3) # try a frontend RefreshState that should timeout as the tablet is busy # running the other one _, stderr = utils.run_vtctl( ['-wait-time', '3s', 'RefreshState', tablet_62344.tablet_alias], expect_fail=True) self.assertIn(protocols_flavor().rpc_timeout_message(), stderr) # wait for the background vtctl bg.wait() if environment.topo_server().flavor() == 'zookeeper': # extra small test: we ran for a while, get the states we were in, # make sure they're accounted for properly # first the query engine States v = utils.get_vars(tablet_62344.port) logging.debug('vars: %s', v) # then the Zookeeper connections if v['ZkCachedConn']['test_nj'] != 'Connected': self.fail('invalid zk test_nj state: %s' % v['ZkCachedConn']['test_nj']) if v['ZkCachedConn']['global'] != 'Connected': self.fail('invalid zk global state: %s' % v['ZkCachedConn']['global']) if v['TabletType'] != 'master': self.fail('TabletType not exported correctly') tablet_62344.kill_vttablet()
def start(self, cell='test_nj', retry_count=2, topo_impl=None, cache_ttl='1s', extra_args=None, tablets=None, tablet_types_to_wait='MASTER,REPLICA', l2vtgates=None, cells_to_watch=None): """Start vtgate. Saves it into the global vtgate variable if not set yet.""" args = environment.binary_args('vtgate') + [ '-port', str(self.port), '-cell', cell, '-retry-count', str(retry_count), '-log_dir', environment.vtlogroot, '-srv_topo_cache_ttl', cache_ttl, '-srv_topo_cache_refresh', cache_ttl, '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-stderrthreshold', get_log_level(), '-normalize_queries', '-gateway_implementation', vtgate_gateway_flavor().flavor(), ] if cells_to_watch: args.extend(vtgate_gateway_flavor().flags(cell=cells_to_watch, tablets=tablets)) else: args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets)) if l2vtgates: args.extend(['-l2vtgate_addrs', ','.join(l2vtgates)]) if tablet_types_to_wait: args.extend(['-tablet_types_to_wait', tablet_types_to_wait]) if protocols_flavor().vtgate_protocol() == 'grpc': args.extend(['-grpc_port', str(self.grpc_port)]) args.extend(['-grpc_max_message_size', str(environment.grpc_max_message_size)]) if protocols_flavor().service_map(): args.extend(['-service_map', ','.join(protocols_flavor().service_map())]) if topo_impl: args.extend(['-topo_implementation', topo_impl]) else: args.extend(environment.topo_server().flags()) if extra_args: args.extend(extra_args) if self.mysql_port: args.extend(['-mysql_server_port', str(self.mysql_port)]) self.proc = run_bg(args) # We use a longer timeout here, as we may be waiting for the initial # state of a few tablets. wait_for_vars('vtgate', self.port, timeout=20.0) global vtgate if not vtgate: vtgate = self
def start(self, cell='test_nj', retry_count=2, topo_impl=None, cache_ttl='1s', extra_args=None, tablets=None, tablet_types_to_wait='MASTER,REPLICA', l2vtgates=None): """Start vtgate. Saves it into the global vtgate variable if not set yet.""" args = environment.binary_args('vtgate') + [ '-port', str(self.port), '-cell', cell, '-retry-count', str(retry_count), '-log_dir', environment.vtlogroot, '-srv_topo_cache_ttl', cache_ttl, '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-stderrthreshold', get_log_level(), '-normalize_queries', ] if l2vtgates: args.extend([ '-gateway_implementation', 'l2vtgategateway', '-l2vtgategateway_addrs', ','.join(l2vtgates), ]) else: args.extend([ '-gateway_implementation', vtgate_gateway_flavor().flavor(), ]) args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets)) if tablet_types_to_wait: args.extend(['-tablet_types_to_wait', tablet_types_to_wait]) if protocols_flavor().vtgate_protocol() == 'grpc': args.extend(['-grpc_port', str(self.grpc_port)]) args.extend(['-grpc_max_message_size', str(environment.grpc_max_message_size)]) if protocols_flavor().service_map(): args.extend(['-service_map', ','.join(protocols_flavor().service_map())]) if topo_impl: args.extend(['-topo_implementation', topo_impl]) else: args.extend(environment.topo_server().flags()) if extra_args: args.extend(extra_args) if self.mysql_port: args.extend(['-mysql_server_port', str(self.mysql_port)]) self.proc = run_bg(args) # We use a longer timeout here, as we may be waiting for the initial # state of a few tablets. wait_for_vars('vtgate', self.port, timeout=20.0) global vtgate if not vtgate: vtgate = self
def start(self, cell='test_nj', retry_count=2, topo_impl=None, cache_ttl='1s', healthcheck_conn_timeout='2s', extra_args=None, tablets=None, tablet_types_to_wait='MASTER,REPLICA', tablet_filters=None): """Start l2vtgate.""" args = environment.binary_args('l2vtgate') + [ '-port', str(self.port), '-cell', cell, '-retry-count', str(retry_count), '-log_dir', environment.vtlogroot, '-srv_topo_cache_ttl', cache_ttl, '-healthcheck_conn_timeout', healthcheck_conn_timeout, '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-gateway_implementation', vtgate_gateway_flavor().flavor(), '-tablet_grpc_combine_begin_execute', ] args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets)) if tablet_types_to_wait: args.extend(['-tablet_types_to_wait', tablet_types_to_wait]) if tablet_filters: args.extend(['-tablet_filters', tablet_filters]) if protocols_flavor().vtgate_protocol() == 'grpc': args.extend(['-grpc_port', str(self.grpc_port)]) if protocols_flavor().service_map(): args.extend( ['-service_map', ','.join(protocols_flavor().service_map())]) if topo_impl: args.extend(['-topo_implementation', topo_impl]) else: args.extend(environment.topo_server().flags()) if extra_args: args.extend(extra_args) self.proc = run_bg(args) wait_for_vars('l2vtgate', self.port)
def start(self, cell='test_nj', retry_count=2, topo_impl=None, cache_ttl='1s', extra_args=None, tablets=None, tablet_types_to_wait='MASTER,REPLICA', tablet_filters=None): """Start l2vtgate.""" args = environment.binary_args('l2vtgate') + [ '-port', str(self.port), '-cell', cell, '-retry-count', str(retry_count), '-log_dir', environment.vtlogroot, '-srv_topo_cache_ttl', cache_ttl, '-srv_topo_cache_refresh', cache_ttl, '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-gateway_implementation', vtgate_gateway_flavor().flavor(), ] args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets)) if tablet_types_to_wait: args.extend(['-tablet_types_to_wait', tablet_types_to_wait]) if tablet_filters: args.extend(['-tablet_filters', tablet_filters]) if protocols_flavor().vtgate_protocol() == 'grpc': args.extend(['-grpc_port', str(self.grpc_port)]) if protocols_flavor().service_map(): args.extend( ['-service_map', ','.join(protocols_flavor().service_map())]) if topo_impl: args.extend(['-topo_implementation', topo_impl]) else: args.extend(environment.topo_server().flags()) if extra_args: args.extend(extra_args) self.proc = run_bg(args) # We use a longer timeout here, as we may be waiting for the initial # state of a few tablets. wait_for_vars('l2vtgate', self.port, timeout=20.0)
def start(self): args = ( environment.binary_args("vtctld") + [ "-debug", "-web_dir", environment.vttop + "/web/vtctld", "--templates", environment.vttop + "/go/cmd/vtctld/templates", "--log_dir", environment.vtlogroot, "--port", str(self.port), "--schema_change_dir", self.schema_change_dir, "--schema_change_controller", "local", "--schema_change_check_interval", "1", "-tablet_manager_protocol", protocols_flavor().tablet_manager_protocol(), "-vtgate_protocol", protocols_flavor().vtgate_protocol(), ] + environment.topo_server().flags() ) if protocols_flavor().service_map(): args.extend(["-service_map", ",".join(protocols_flavor().service_map())]) if protocols_flavor().vtctl_client_protocol() == "grpc": args.extend(["-grpc_port", str(self.grpc_port)]) stderr_fd = open(os.path.join(environment.tmproot, "vtctld.stderr"), "w") self.proc = run_bg(args, stderr=stderr_fd) # wait for the process to listen to RPC timeout = 30 while True: v = get_vars(self.port) if v: break timeout = wait_step("waiting for vtctld to start", timeout, sleep_time=0.2) # save the running instance so vtctl commands can be remote executed now global vtctld, vtctld_connection if not vtctld: vtctld = self protocol, endpoint = self.rpc_endpoint(python=True) vtctld_connection = vtctl_client.connect(protocol, endpoint, 30) return self.proc
def start(self, cell='test_nj', retry_count=2, topo_impl=None, cache_ttl='1s', healthcheck_conn_timeout='2s', extra_args=None, tablets=None, tablet_types_to_wait='MASTER,REPLICA', l2vtgates=None): """Start vtgate. Saves it into the global vtgate variable if not set yet.""" args = environment.binary_args('vtgate') + [ '-port', str(self.port), '-cell', cell, '-retry-count', str(retry_count), '-log_dir', environment.vtlogroot, '-srv_topo_cache_ttl', cache_ttl, '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-tablet_grpc_combine_begin_execute', ] if l2vtgates: args.extend([ '-gateway_implementation', 'l2vtgategateway', '-l2vtgategateway_addrs', ','.join(l2vtgates), ]) else: args.extend([ '-healthcheck_conn_timeout', healthcheck_conn_timeout, '-gateway_implementation', vtgate_gateway_flavor().flavor(), ]) args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets)) if tablet_types_to_wait: args.extend(['-tablet_types_to_wait', tablet_types_to_wait]) if protocols_flavor().vtgate_protocol() == 'grpc': args.extend(['-grpc_port', str(self.grpc_port)]) if protocols_flavor().service_map(): args.extend(['-service_map', ','.join(protocols_flavor().service_map())]) if topo_impl: args.extend(['-topo_implementation', topo_impl]) else: args.extend(environment.topo_server().flags()) if extra_args: args.extend(extra_args) self.proc = run_bg(args) wait_for_vars('vtgate', self.port) global vtgate if not vtgate: vtgate = self
def __init__(self, port=None): """Creates the Vtgate instance and reserve the ports if necessary.""" self.port = port or environment.reserve_ports(1) if protocols_flavor().vtgate_protocol() == 'grpc': self.grpc_port = environment.reserve_ports(1) self.secure_port = None self.proc = None
def setUp(self): addr = 'localhost:%d' % vtgateclienttest_port protocol = protocols_flavor().vtgate_python_protocol() self.conn = vtgate_client.connect(protocol, addr, 30.0) logging.info( 'Start: %s, protocol %s.', '.'.join(self.id().split('.')[-2:]), protocol)
def setUp(self): super(TestPythonClientBase, self).setUp() addr = 'localhost:%d' % vtgateclienttest_port protocol = protocols_flavor().vtgate_python_protocol() self.conn = vtgate_client.connect(protocol, addr, 30.0) logging.info('Start: %s, protocol %s.', '.'.join(self.id().split('.')[-2:]), protocol)
def vtclient2(uid, path, query, bindvars=None, user=None, password=None, driver=None, verbose=False, raise_on_error=True): if (user is None) != (password is None): raise TypeError("you should provide either both or none of user and password") # for ZK paths to not have // in the path, that confuses things if path.startswith('/'): path = path[1:] server = "localhost:%u/%s" % (uid, path) cmdline = environment.binary_args('vtclient2') + ['-server', server] cmdline += environment.topo_server().flags() cmdline += protocols_flavor().tabletconn_protocol_flags() if user is not None: cmdline.extend(['-tablet-bson-username', user, '-tablet-bson-password', password]) if bindvars: cmdline.extend(['-bindvars', bindvars]) if driver: cmdline.extend(['-driver', driver]) if verbose: cmdline.extend(['-alsologtostderr', '-verbose']) cmdline.append(query) return run(cmdline, raise_on_error=raise_on_error, trap_output=True)
def run_vtworker_client(args, rpc_port): """Runs vtworkerclient to execute a command on a remote vtworker. Args: args: Atr string to send to binary. rpc_port: Port number. Returns: out: stdout of the vtworkerclient invocation err: stderr of the vtworkerclient invocation """ out, err = run( environment.binary_args("vtworkerclient") + [ "-vtworker_client_protocol", protocols_flavor().vtworker_client_protocol(), "-server", "localhost:%d" % rpc_port, "-stderrthreshold", get_log_level(), ] + args, trap_output=True, ) return out, err
def vtclient(self, sql, tablet_type='master', bindvars=None, streaming=False, verbose=False, raise_on_error=False): """Uses the vtclient binary to send a query to vtgate.""" args = environment.binary_args('vtclient') + [ '-server', self.rpc_endpoint(), '-tablet_type', tablet_type, '-vtgate_protocol', protocols_flavor().vtgate_protocol() ] if bindvars: args.extend(['-bind_variables', json.dumps(bindvars)]) if streaming: args.append('-streaming') if verbose: args.append('-alsologtostderr') args.append(sql) out, err = run(args, raise_on_error=raise_on_error, trap_output=True) out = out.splitlines() return out, err
def _read_srv_keyspace(self, keyspace_name): addr = utils.vtgate.rpc_endpoint() protocol = protocols_flavor().vtgate_python_protocol() conn = vtgate_client.connect(protocol, addr, 30.0) result = conn.get_srv_keyspace(keyspace_name) conn.close() return result
def test_streaming_transient_error(self): """Test we raise dbexceptions.IntegrityError for StreamExecute calls.""" # TODO(aaijazi): this test doesn't work for all clients yet. if protocols_flavor().vtgate_python_protocol() != 'gorpc': return self._verify_exception_for_stream_execute('error://transient error', dbexceptions.TransientError)
def run_vtworker(clargs, log_level='', auto_log=False, expect_fail=False, **kwargs): args = environment.binary_args('vtworker') + [ '-log_dir', environment.vtlogroot, '-port', str(environment.reserve_ports(1)) ] args.extend(environment.topo_server().flags()) args.extend(protocols_flavor().tablet_manager_protocol_flags()) if auto_log: if options.verbose == 2: log_level = 'INFO' elif options.verbose == 1: log_level = 'WARNING' else: log_level = 'ERROR' if log_level: args.append('--stderrthreshold=%s' % log_level) cmd = args + clargs if expect_fail: return run_fail(cmd, **kwargs) return run(cmd, **kwargs)
def _get_vtworker_cmd(clargs, log_level='', auto_log=False): """Assembles the command that is needed to run a vtworker. Returns: cmd - list of cmd arguments, can be passed to any `run`-like functions port - int with the port number that the vtworker is running with """ port = environment.reserve_ports(1) args = environment.binary_args('vtworker') + [ '-log_dir', environment.vtlogroot, '-min_healthy_rdonly_endpoints', '1', '-port', str(port), '-resolve_ttl', '2s', '-executefetch_retry_time', '1s', ] args.extend(environment.topo_server().flags()) args.extend(protocols_flavor().tablet_manager_protocol_flags()) if auto_log: if options.verbose == 2: log_level='INFO' elif options.verbose == 1: log_level='WARNING' else: log_level='ERROR' if log_level: args.append('--stderrthreshold=%s' % log_level) cmd = args + clargs return cmd, port
def vtclient2(uid, path, query, bindvars=None, user=None, password=None, driver=None, verbose=False, raise_on_error=True): if (user is None) != (password is None): raise TypeError( "you should provide either both or none of user and password") # for ZK paths to not have // in the path, that confuses things if path.startswith('/'): path = path[1:] server = "localhost:%u/%s" % (uid, path) cmdline = environment.binary_args('vtclient2') + ['-server', server] cmdline += environment.topo_server().flags() cmdline += protocols_flavor().tabletconn_protocol_flags() if user is not None: cmdline.extend( ['-tablet-bson-username', user, '-tablet-bson-password', password]) if bindvars: cmdline.extend(['-bindvars', bindvars]) if driver: cmdline.extend(['-driver', driver]) if verbose: cmdline.extend(['-alsologtostderr', '-verbose']) cmdline.append(query) return run(cmdline, raise_on_error=raise_on_error, trap_output=True)
def get_connection(timeout=10.0): protocol = protocols_flavor().vtgate_python_protocol() try: return vtgate_client.connect(protocol, utils.vtgate.addr(), timeout) except Exception: logging.exception('Connection to vtgate (timeout=%s) failed.', timeout) raise
def _get_vtworker_cmd(clargs, auto_log=False): """Assembles the command that is needed to run a vtworker. Args: clargs: Command line arguments passed to vtworker. auto_log: If true, set --stderrthreshold according to the test log level. Returns: cmd - list of cmd arguments, can be passed to any `run`-like functions port - int with the port number that the vtworker is running with rpc_port - int with the port number of the RPC interface """ port = environment.reserve_ports(1) rpc_port = port args = environment.binary_args('vtworker') + [ '-log_dir', environment.vtlogroot, '-min_healthy_rdonly_endpoints', '1', '-port', str(port), # use a long resolve TTL because of potential race conditions with doing # an EmergencyReparent and resolving the master (as EmergencyReparent # will delete the old master before updating the shard record with the # new master) '-resolve_ttl', '10s', '-executefetch_retry_time', '1s', '-tablet_manager_protocol', protocols_flavor().tablet_manager_protocol(), '-tablet_protocol', protocols_flavor().tabletconn_protocol(), ] args.extend(environment.topo_server().flags()) if protocols_flavor().service_map(): args.extend( ['-service_map', ','.join(protocols_flavor().service_map())]) if protocols_flavor().vtworker_client_protocol() == 'grpc': rpc_port = environment.reserve_ports(1) args.extend(['-grpc_port', str(rpc_port)]) if auto_log: args.append('--stderrthreshold=%s' % get_log_level()) cmd = args + clargs return cmd, port, rpc_port
def setUp(self): self.master_tablet = shard_1_master if protocols_flavor().vtgate_python_types() == 'proto3': self.int_type = 265 self.string_type = 6165 else: self.int_type = 8L self.string_type = 253L
def test_transient_error(self): """Test we raise dbexceptions.TransientError for Execute calls.""" # TODO(aaijazi): this test doesn't work for all clients yet. if protocols_flavor().vtgate_python_protocol() != 'gorpc': return # Special query that makes vtgateclienttest return a TransientError. self._verify_exception_for_execute('error://transient error', dbexceptions.TransientError)
def _get_vtworker_cmd(clargs, auto_log=False): """Assembles the command that is needed to run a vtworker. Args: clargs: Command line arguments passed to vtworker. auto_log: If true, set --stderrthreshold according to the test log level. Returns: cmd - list of cmd arguments, can be passed to any `run`-like functions port - int with the port number that the vtworker is running with rpc_port - int with the port number of the RPC interface """ port = environment.reserve_ports(1) rpc_port = port args = environment.binary_args("vtworker") + [ "-log_dir", environment.vtlogroot, "-min_healthy_rdonly_endpoints", "1", "-port", str(port), # use a long resolve TTL because of potential race conditions with doing # an EmergencyReparent and resolving the master (as EmergencyReparent # will delete the old master before updating the shard record with the # new master) "-resolve_ttl", "10s", "-executefetch_retry_time", "1s", "-tablet_manager_protocol", protocols_flavor().tablet_manager_protocol(), "-tablet_protocol", protocols_flavor().tabletconn_protocol(), ] args.extend(environment.topo_server().flags()) if protocols_flavor().service_map(): args.extend(["-service_map", ",".join(protocols_flavor().service_map())]) if protocols_flavor().vtworker_client_protocol() == "grpc": rpc_port = environment.reserve_ports(1) args.extend(["-grpc_port", str(rpc_port)]) if auto_log: args.append("--stderrthreshold=%s" % get_log_level()) cmd = args + clargs return cmd, port, rpc_port
def run_automation_server(auto_log=False): """Starts a background automation_server process. Returns: rpc_port - int with the port number of the RPC interface """ rpc_port = environment.reserve_ports(1) args = environment.binary_args('automation_server') + [ '-log_dir', environment.vtlogroot, '-port', str(rpc_port), '-vtctl_client_protocol', protocols_flavor().vtctl_client_protocol(), '-vtworker_client_protocol', protocols_flavor().vtworker_client_protocol(), ] if auto_log: args.append('--stderrthreshold=%s' % get_log_level()) return run_bg(args), rpc_port
def start(self, cell='test_nj', retry_delay=1, retry_count=2, topo_impl=None, cache_ttl='1s', auth=False, timeout_total='4s', timeout_per_conn='2s', extra_args=None): """Starts the process for this vtgate instance. If no other instance has been started, saves it into the global vtgate variable. """ args = environment.binary_args('vtgate') + [ '-port', str(self.port), '-cell', cell, '-retry-delay', '%ss' % (str(retry_delay)), '-retry-count', str(retry_count), '-log_dir', environment.vtlogroot, '-srv_topo_cache_ttl', cache_ttl, '-conn-timeout-total', timeout_total, '-conn-timeout-per-conn', timeout_per_conn, '-bsonrpc_timeout', '5s', '-tablet_protocol', protocols_flavor().tabletconn_protocol(), ] if protocols_flavor().vtgate_protocol() == 'grpc': args.extend(['-grpc_port', str(self.grpc_port)]) if protocols_flavor().service_map(): args.extend(['-service_map', ','.join(protocols_flavor().service_map())]) if topo_impl: args.extend(['-topo_implementation', topo_impl]) else: args.extend(environment.topo_server().flags()) if auth: args.extend(['-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json')]) if extra_args: args.extend(extra_args) self.proc = run_bg(args) if self.secure_port: wait_for_vars('vtgate', self.port, 'SecureConnections') else: wait_for_vars('vtgate', self.port) global vtgate if not vtgate: vtgate = self
def test_actions_and_timeouts(self): # Start up a master mysql and vttablet utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) tablet_62344.init_tablet('master', 'test_keyspace', '0') tablet_62344.create_db('vt_test_keyspace') tablet_62344.start_vttablet() # validate topology after starting tablet so that tablet has a chance # to update shard master_alias timeout = 10 while True: shard = utils.run_vtctl_json(['GetShard', 'test_keyspace/0']) if shard['master_alias']['uid'] == 62344: break wait_step('master_alias has been set', timeout) utils.validate_topology() utils.run_vtctl(['Ping', tablet_62344.tablet_alias]) # schedule long action in the background, sleep a little bit to make sure # it started to run args = (environment.binary_args('vtctl') + environment.topo_server().flags() + [ '-tablet_manager_protocol', protocols_flavor().tablet_manager_protocol(), '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-log_dir', environment.vtlogroot, 'Sleep', tablet_62344.tablet_alias, '10s' ]) bg = utils.run_bg(args) time.sleep(3) # try a frontend RefreshState that should timeout as the tablet is busy # running the other one _, stderr = utils.run_vtctl( ['-wait-time', '3s', 'RefreshState', tablet_62344.tablet_alias], expect_fail=True) self.assertIn(protocols_flavor().rpc_timeout_message(), stderr) # wait for the background vtctl bg.wait() tablet_62344.kill_vttablet()
def start(self, enable_schema_change_dir=False): args = environment.binary_args('vtctld') + [ '-enable_queries', '-cell', 'test_nj', '-web_dir', environment.vttop + '/web/vtctld', '--log_dir', environment.vtlogroot, '--port', str(self.port), '-tablet_manager_protocol', protocols_flavor().tablet_manager_protocol(), '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-throttler_client_protocol', protocols_flavor().throttler_client_protocol(), '-vtgate_protocol', protocols_flavor().vtgate_protocol(), ] + environment.topo_server().flags() if enable_schema_change_dir: args += [ '--schema_change_dir', self.schema_change_dir, '--schema_change_controller', 'local', '--schema_change_check_interval', '1', ] if protocols_flavor().service_map(): args.extend( ['-service_map', ','.join(protocols_flavor().service_map())]) if protocols_flavor().vtctl_client_protocol() == 'grpc': args.extend(['-grpc_port', str(self.grpc_port)]) stdout_fd = open(os.path.join(environment.tmproot, 'vtctld.stdout'), 'w') stderr_fd = open(os.path.join(environment.tmproot, 'vtctld.stderr'), 'w') self.proc = run_bg(args, stdout=stdout_fd, stderr=stderr_fd) # wait for the process to listen to RPC timeout = 30 while True: v = get_vars(self.port) if v: break if self.proc.poll() is not None: raise TestError('vtctld died while starting') timeout = wait_step('waiting for vtctld to start', timeout, sleep_time=0.2) # save the running instance so vtctl commands can be remote executed now global vtctld, vtctld_connection if not vtctld: vtctld = self protocol, endpoint = self.rpc_endpoint(python=True) vtctld_connection = vtctl_client.connect(protocol, endpoint, 30) return self.proc
def vtctl_client(self, args): if options.verbose == 2: log_level='INFO' elif options.verbose == 1: log_level='WARNING' else: log_level='ERROR' port = self.port if protocols_flavor().vtctl_client_protocol() == 'grpc': port = self.grpc_port out, err = run(environment.binary_args('vtctlclient') + ['-vtctl_client_protocol', protocols_flavor().vtctl_client_protocol(), '-server', 'localhost:%u' % port, '-stderrthreshold', log_level] + args, trap_output=True) return out
def vtctl_client(self, args): if options.verbose == 2: log_level = 'INFO' elif options.verbose == 1: log_level = 'WARNING' else: log_level = 'ERROR' port = self.port if protocols_flavor().vtctl_client_protocol() == 'grpc': port = self.grpc_port out, err = run(environment.binary_args('vtctlclient') + [ '-vtctl_client_protocol', protocols_flavor().vtctl_client_protocol(), '-server', 'localhost:%u' % port, '-stderrthreshold', log_level ] + args, trap_output=True) return out
def _get_vtworker_cmd(clargs, log_level='', auto_log=False): """Assembles the command that is needed to run a vtworker. Returns: cmd - list of cmd arguments, can be passed to any `run`-like functions port - int with the port number that the vtworker is running with """ port = environment.reserve_ports(1) rpc_port = environment.reserve_ports(1) args = environment.binary_args('vtworker') + [ '-log_dir', environment.vtlogroot, '-min_healthy_rdonly_endpoints', '1', '-port', str(port), '-resolve_ttl', '2s', '-executefetch_retry_time', '1s', ] args.extend(environment.topo_server().flags()) args.extend([ '-tablet_manager_protocol', protocols_flavor().tablet_manager_protocol() ]) if protocols_flavor().service_map(): args.extend( ['-service_map', ",".join(protocols_flavor().service_map())]) if protocols_flavor().vtworker_client_protocol() == 'grpc': args.extend(['-grpc_port', str(rpc_port)]) if auto_log: if options.verbose == 2: log_level = 'INFO' elif options.verbose == 1: log_level = 'WARNING' else: log_level = 'ERROR' if log_level: args.append('--stderrthreshold=%s' % log_level) cmd = args + clargs return cmd, port, rpc_port
def update_stream_python_endpoint(self): protocol = protocols_flavor().binlog_player_python_protocol() port = self.port if protocol == "gorpc": from vtdb import gorpc_update_stream elif protocol == "grpc": # import the grpc update stream client implementation, change the port from vtdb import grpc_update_stream port = self.grpc_port return (protocol, 'localhost:%u' % port)
def start(self, enable_schema_change_dir=False): # Note the vtctld2 web dir is set to 'dist', which is populated # when a toplevel 'make build_web' is run. This is meant to test # the development version of the UI. The real checked-in app is in # app/. args = environment.binary_args('vtctld') + [ '-enable_queries', '-cell', 'test_nj', '-web_dir', environment.vttop + '/web/vtctld', '-web_dir2', environment.vttop + '/web/vtctld2/dist', '--log_dir', environment.vtlogroot, '--port', str(self.port), '-tablet_manager_protocol', protocols_flavor().tablet_manager_protocol(), '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-throttler_client_protocol', protocols_flavor().throttler_client_protocol(), '-vtgate_protocol', protocols_flavor().vtgate_protocol(), '-workflow_manager_init', '-workflow_manager_use_election', ] + environment.topo_server().flags() # TODO(b/26388813): Remove the next two lines once vtctl WaitForDrain is # integrated in the vtctl MigrateServed* commands. args.extend(['--wait_for_drain_sleep_rdonly', '0s']) args.extend(['--wait_for_drain_sleep_replica', '0s']) if enable_schema_change_dir: args += [ '--schema_change_dir', self.schema_change_dir, '--schema_change_controller', 'local', '--schema_change_check_interval', '1', ] if protocols_flavor().service_map(): args.extend(['-service_map', ','.join(protocols_flavor().service_map())]) if protocols_flavor().vtctl_client_protocol() == 'grpc': args.extend(['-grpc_port', str(self.grpc_port)]) stdout_fd = open(os.path.join(environment.tmproot, 'vtctld.stdout'), 'w') stderr_fd = open(os.path.join(environment.tmproot, 'vtctld.stderr'), 'w') self.proc = run_bg(args, stdout=stdout_fd, stderr=stderr_fd) # wait for the process to listen to RPC timeout = 30 while True: v = get_vars(self.port) if v: break if self.proc.poll() is not None: raise TestError('vtctld died while starting') timeout = wait_step('waiting for vtctld to start', timeout, sleep_time=0.2) # save the running instance so vtctl commands can be remote executed now global vtctld, vtctld_connection if not vtctld: vtctld = self protocol, endpoint = self.rpc_endpoint(python=True) vtctld_connection = vtctl_client.connect(protocol, endpoint, 30) return self.proc
def start(self): args = environment.binary_args('vtctld') + [ '-debug', '-templates', environment.vttop + '/go/cmd/vtctld/templates', '-log_dir', environment.vtlogroot, '-port', str(self.port), ] + \ environment.topo_server().flags() + \ protocols_flavor().tablet_manager_protocol_flags() if protocols_flavor().vtctl_client_protocol() == "grpc": args += [ '-grpc_port', str(self.grpc_port), '-service_map', 'grpc-vtctl' ] stderr_fd = open(os.path.join(environment.tmproot, "vtctld.stderr"), "w") self.proc = run_bg(args, stderr=stderr_fd) # wait for the process to listen to RPC timeout = 30 while True: v = get_vars(self.port) if v: break timeout = wait_step('waiting for vtctld to start', timeout, sleep_time=0.2) # save the running instance so vtctl commands can be remote executed now protocol = protocols_flavor().vtctl_client_protocol() if protocol == "grpc": # import the grpc vtctl client implementation, disabled for now: # from vtctl import grpc_vtctl_client # temporary protocol override until python client support works protocol = "gorpc" global vtctld, vtctld_connection if not vtctld: vtctld = self vtctld_connection = vtctl_client.connect( protocol, 'localhost:%u' % self.port, 30) return self.proc
def run_vtctl_vtctl(clargs, auto_log=False, expect_fail=False, **kwargs): args = environment.binary_args('vtctl') + ['-log_dir', environment.vtlogroot] args.extend(environment.topo_server().flags()) args.extend(['-tablet_manager_protocol', protocols_flavor().tablet_manager_protocol()]) args.extend(['-tablet_protocol', protocols_flavor().tabletconn_protocol()]) args.extend(['-vtgate_protocol', protocols_flavor().vtgate_protocol()]) if auto_log: args.append('--stderrthreshold=%s' % get_log_level()) if isinstance(clargs, str): cmd = ' '.join(args) + ' ' + clargs else: cmd = args + clargs if expect_fail: return run_fail(cmd, **kwargs) return run(cmd, **kwargs)
def start(self): args = environment.binary_args('vtctld') + [ '-debug', '--templates', environment.vttop + '/go/cmd/vtctld/templates', '--log_dir', environment.vtlogroot, '--port', str(self.port), '--schema_change_dir', self.schema_change_dir, '--schema_change_controller', 'local', '--schema_change_check_interval', '1', '-tablet_manager_protocol', protocols_flavor().tablet_manager_protocol(), ] + \ environment.topo_server().flags() + \ protocols_flavor().vtgate_protocol_flags() if protocols_flavor().service_map(): args.extend( ['-service_map', ",".join(protocols_flavor().service_map())]) if protocols_flavor().vtctl_client_protocol() == 'grpc': args.extend(['-grpc_port', str(self.grpc_port)]) stderr_fd = open(os.path.join(environment.tmproot, 'vtctld.stderr'), 'w') self.proc = run_bg(args, stderr=stderr_fd) # wait for the process to listen to RPC timeout = 30 while True: v = get_vars(self.port) if v: break timeout = wait_step('waiting for vtctld to start', timeout, sleep_time=0.2) # save the running instance so vtctl commands can be remote executed now global vtctld, vtctld_connection if not vtctld: vtctld = self protocol, endpoint = self.rpc_endpoint(python=True) vtctld_connection = vtctl_client.connect(protocol, endpoint, 30) return self.proc
def rpc_endpoint(self, python=False): """RPC endpoint to vtctld. The RPC endpoint may differ from the webinterface URL e.g. because gRPC requires a dedicated port. Returns: protocol - string e.g. 'grpc' endpoint - string e.g. 'localhost:15001' """ if python: protocol = protocols_flavor().vtctl_python_client_protocol() else: protocol = protocols_flavor().vtctl_client_protocol() rpc_port = self.port if protocol == 'grpc': # import the grpc vtctl client implementation, change the port if python: from vtctl import grpc_vtctl_client rpc_port = self.grpc_port return (protocol, '%s:%d' % (socket.getfqdn(), rpc_port))
def rpc_endpoint(self, python=False): """RPC endpoint to vtctld. The RPC endpoint may differ from the webinterface URL e.g. because gRPC requires a dedicated port. Args: python: boolean, True iff this is for access with Python (as opposed to Go). Returns: protocol - string e.g. 'grpc' endpoint - string e.g. 'localhost:15001' """ if python: protocol = protocols_flavor().vtctl_python_client_protocol() else: protocol = protocols_flavor().vtctl_client_protocol() rpc_port = self.port if protocol == 'grpc': rpc_port = self.grpc_port return (protocol, '%s:%d' % (socket.getfqdn(), rpc_port))