def start(self, cell='test_nj', retry_delay=1, retry_count=2, topo_impl=None, cache_ttl='1s', timeout_total='2s', timeout_per_conn='1s', extra_args=None, tablets=None): """Start vtgate. Saves it into the global vtgate variable if not set yet.""" args = environment.binary_args('vtgate') + [ '-port', str(self.port), '-cell', cell, '-retry-delay', '%ss' % (str(retry_delay)), '-retry-count', str(retry_count), '-log_dir', environment.vtlogroot, '-srv_topo_cache_ttl', cache_ttl, '-conn-timeout-total', timeout_total, '-conn-timeout-per-conn', timeout_per_conn, '-bsonrpc_timeout', '5s', '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-gateway_implementation', vtgate_gateway_flavor().flavor(), '-tablet_types_to_wait', 'MASTER,REPLICA', ] args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets)) if protocols_flavor().vtgate_protocol() == 'grpc': args.extend(['-grpc_port', str(self.grpc_port)]) if protocols_flavor().service_map(): args.extend( ['-service_map', ','.join(protocols_flavor().service_map())]) if topo_impl: args.extend(['-topo_implementation', topo_impl]) else: args.extend(environment.topo_server().flags()) if extra_args: args.extend(extra_args) self.proc = run_bg(args) if self.secure_port: wait_for_vars('vtgate', self.port, 'SecureConnections') else: wait_for_vars('vtgate', self.port) global vtgate if not vtgate: vtgate = self
def setup_vtgate(port=None, extra_args=None): utils.VtGate(port=port).start(extra_args=extra_args, tablets=[shard_0_master, shard_0_replica1]) if vtgate_gateway_flavor().flavor() != 'shardgateway': wait_for_vars(vtgate_gateway_flavor().connection_count_vars(), '%s.%s.master' % (KEYSPACE_NAME, SHARD_NAMES[0]), 1) wait_for_vars(vtgate_gateway_flavor().connection_count_vars(), '%s.%s.replica' % (KEYSPACE_NAME, SHARD_NAMES[0]), 1)
def start( self, cell="test_nj", retry_delay=1, retry_count=2, topo_impl=None, cache_ttl="1s", timeout_total="2s", timeout_per_conn="1s", extra_args=None, tablets=None, ): """Start vtgate. Saves it into the global vtgate variable if not set yet.""" args = environment.binary_args("vtgate") + [ "-port", str(self.port), "-cell", cell, "-retry-delay", "%ss" % (str(retry_delay)), "-retry-count", str(retry_count), "-log_dir", environment.vtlogroot, "-srv_topo_cache_ttl", cache_ttl, "-conn-timeout-total", timeout_total, "-conn-timeout-per-conn", timeout_per_conn, "-bsonrpc_timeout", "5s", "-tablet_protocol", protocols_flavor().tabletconn_protocol(), "-gateway_implementation", vtgate_gateway_flavor().flavor(), "-tablet_types_to_wait", "MASTER,REPLICA", ] args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets)) if protocols_flavor().vtgate_protocol() == "grpc": args.extend(["-grpc_port", str(self.grpc_port)]) if protocols_flavor().service_map(): args.extend(["-service_map", ",".join(protocols_flavor().service_map())]) if topo_impl: args.extend(["-topo_implementation", topo_impl]) else: args.extend(environment.topo_server().flags()) if extra_args: args.extend(extra_args) self.proc = run_bg(args) wait_for_vars("vtgate", self.port) global vtgate if not vtgate: vtgate = self
def start(self, cell='test_nj', retry_count=2, topo_impl=None, cache_ttl='1s', extra_args=None, tablets=None, tablet_types_to_wait='MASTER,REPLICA', l2vtgates=None, cells_to_watch=None): """Start vtgate. Saves it into the global vtgate variable if not set yet.""" args = environment.binary_args('vtgate') + [ '-port', str(self.port), '-cell', cell, '-retry-count', str(retry_count), '-log_dir', environment.vtlogroot, '-srv_topo_cache_ttl', cache_ttl, '-srv_topo_cache_refresh', cache_ttl, '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-stderrthreshold', get_log_level(), '-normalize_queries', '-gateway_implementation', vtgate_gateway_flavor().flavor(), ] if cells_to_watch: args.extend(vtgate_gateway_flavor().flags(cell=cells_to_watch, tablets=tablets)) else: args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets)) if l2vtgates: args.extend(['-l2vtgate_addrs', ','.join(l2vtgates)]) if tablet_types_to_wait: args.extend(['-tablet_types_to_wait', tablet_types_to_wait]) if protocols_flavor().vtgate_protocol() == 'grpc': args.extend(['-grpc_port', str(self.grpc_port)]) args.extend(['-grpc_max_message_size', str(environment.grpc_max_message_size)]) if protocols_flavor().service_map(): args.extend(['-service_map', ','.join(protocols_flavor().service_map())]) if topo_impl: args.extend(['-topo_implementation', topo_impl]) else: args.extend(environment.topo_server().flags()) if extra_args: args.extend(extra_args) if self.mysql_port: args.extend(['-mysql_server_port', str(self.mysql_port)]) self.proc = run_bg(args) # We use a longer timeout here, as we may be waiting for the initial # state of a few tablets. wait_for_vars('vtgate', self.port, timeout=20.0) global vtgate if not vtgate: vtgate = self
def start(self, cell='test_nj', retry_count=2, topo_impl=None, cache_ttl='1s', extra_args=None, tablets=None, tablet_types_to_wait='MASTER,REPLICA', l2vtgates=None): """Start vtgate. Saves it into the global vtgate variable if not set yet.""" args = environment.binary_args('vtgate') + [ '-port', str(self.port), '-cell', cell, '-retry-count', str(retry_count), '-log_dir', environment.vtlogroot, '-srv_topo_cache_ttl', cache_ttl, '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-stderrthreshold', get_log_level(), '-normalize_queries', ] if l2vtgates: args.extend([ '-gateway_implementation', 'l2vtgategateway', '-l2vtgategateway_addrs', ','.join(l2vtgates), ]) else: args.extend([ '-gateway_implementation', vtgate_gateway_flavor().flavor(), ]) args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets)) if tablet_types_to_wait: args.extend(['-tablet_types_to_wait', tablet_types_to_wait]) if protocols_flavor().vtgate_protocol() == 'grpc': args.extend(['-grpc_port', str(self.grpc_port)]) args.extend(['-grpc_max_message_size', str(environment.grpc_max_message_size)]) if protocols_flavor().service_map(): args.extend(['-service_map', ','.join(protocols_flavor().service_map())]) if topo_impl: args.extend(['-topo_implementation', topo_impl]) else: args.extend(environment.topo_server().flags()) if extra_args: args.extend(extra_args) if self.mysql_port: args.extend(['-mysql_server_port', str(self.mysql_port)]) self.proc = run_bg(args) # We use a longer timeout here, as we may be waiting for the initial # state of a few tablets. wait_for_vars('vtgate', self.port, timeout=20.0) global vtgate if not vtgate: vtgate = self
def wait_for_endpoints(self, name, count, timeout=20.0): """waits until vtgate gets endpoints. Args: name: name of the endpoint, in the form: 'keyspace.shard.type'. count: how many endpoints to wait for. timeout: how long to wait. """ if vtgate_gateway_flavor().flavor() == 'shardgateway': return wait_for_vars('vtgate', self.port, var=vtgate_gateway_flavor().connection_count_vars(), key=name, value=count, timeout=timeout)
def setup_vtgate(port=None, extra_args=None): utils.VtGate(port=port).start( extra_args=extra_args, tablets=[shard_0_master, shard_0_replica1]) if vtgate_gateway_flavor().flavor() != 'shardgateway': wait_for_vars( vtgate_gateway_flavor().connection_count_vars(), '%s.%s.master' % (KEYSPACE_NAME, SHARD_NAMES[0]), 1) wait_for_vars( vtgate_gateway_flavor().connection_count_vars(), '%s.%s.replica' % (KEYSPACE_NAME, SHARD_NAMES[0]), 1)
def start(self, cell='test_nj', retry_count=2, topo_impl=None, cache_ttl='1s', healthcheck_conn_timeout='2s', extra_args=None, tablets=None, tablet_types_to_wait='MASTER,REPLICA'): """Start vtgate. Saves it into the global vtgate variable if not set yet.""" args = environment.binary_args('vtgate') + [ '-port', str(self.port), '-cell', cell, '-retry-count', str(retry_count), '-log_dir', environment.vtlogroot, '-srv_topo_cache_ttl', cache_ttl, '-healthcheck_conn_timeout', healthcheck_conn_timeout, '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-gateway_implementation', vtgate_gateway_flavor().flavor(), '-tablet_grpc_combine_begin_execute', ] args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets)) if tablet_types_to_wait: args.extend(['-tablet_types_to_wait', tablet_types_to_wait]) if protocols_flavor().vtgate_protocol() == 'grpc': args.extend(['-grpc_port', str(self.grpc_port)]) if protocols_flavor().service_map(): args.extend( ['-service_map', ','.join(protocols_flavor().service_map())]) if topo_impl: args.extend(['-topo_implementation', topo_impl]) else: args.extend(environment.topo_server().flags()) if extra_args: args.extend(extra_args) self.proc = run_bg(args) wait_for_vars('vtgate', self.port) global vtgate if not vtgate: vtgate = self
def start(self, cell='test_nj', retry_count=2, topo_impl=None, cache_ttl='1s', extra_args=None, tablets=None, tablet_types_to_wait='MASTER,REPLICA', tablet_filters=None): """Start l2vtgate.""" args = environment.binary_args('l2vtgate') + [ '-port', str(self.port), '-cell', cell, '-retry-count', str(retry_count), '-log_dir', environment.vtlogroot, '-srv_topo_cache_ttl', cache_ttl, '-srv_topo_cache_refresh', cache_ttl, '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-gateway_implementation', vtgate_gateway_flavor().flavor(), ] args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets)) if tablet_types_to_wait: args.extend(['-tablet_types_to_wait', tablet_types_to_wait]) if tablet_filters: args.extend(['-tablet_filters', tablet_filters]) if protocols_flavor().vtgate_protocol() == 'grpc': args.extend(['-grpc_port', str(self.grpc_port)]) if protocols_flavor().service_map(): args.extend( ['-service_map', ','.join(protocols_flavor().service_map())]) if topo_impl: args.extend(['-topo_implementation', topo_impl]) else: args.extend(environment.topo_server().flags()) if extra_args: args.extend(extra_args) self.proc = run_bg(args) # We use a longer timeout here, as we may be waiting for the initial # state of a few tablets. wait_for_vars('l2vtgate', self.port, timeout=20.0)
def start(self, cell='test_nj', retry_count=2, topo_impl=None, cache_ttl='1s', healthcheck_conn_timeout='2s', extra_args=None, tablets=None, tablet_types_to_wait='MASTER,REPLICA', l2vtgates=None): """Start vtgate. Saves it into the global vtgate variable if not set yet.""" args = environment.binary_args('vtgate') + [ '-port', str(self.port), '-cell', cell, '-retry-count', str(retry_count), '-log_dir', environment.vtlogroot, '-srv_topo_cache_ttl', cache_ttl, '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-tablet_grpc_combine_begin_execute', ] if l2vtgates: args.extend([ '-gateway_implementation', 'l2vtgategateway', '-l2vtgategateway_addrs', ','.join(l2vtgates), ]) else: args.extend([ '-healthcheck_conn_timeout', healthcheck_conn_timeout, '-gateway_implementation', vtgate_gateway_flavor().flavor(), ]) args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets)) if tablet_types_to_wait: args.extend(['-tablet_types_to_wait', tablet_types_to_wait]) if protocols_flavor().vtgate_protocol() == 'grpc': args.extend(['-grpc_port', str(self.grpc_port)]) if protocols_flavor().service_map(): args.extend(['-service_map', ','.join(protocols_flavor().service_map())]) if topo_impl: args.extend(['-topo_implementation', topo_impl]) else: args.extend(environment.topo_server().flags()) if extra_args: args.extend(extra_args) self.proc = run_bg(args) wait_for_vars('vtgate', self.port) global vtgate if not vtgate: vtgate = self
def start(self, cell='test_nj', retry_delay=1, retry_count=2, topo_impl=None, cache_ttl='1s', timeout_total='2s', timeout_per_conn='1s', extra_args=None): """Start vtgate. Saves it into the global vtgate variable if not set yet.""" args = environment.binary_args('vtgate') + [ '-port', str(self.port), '-cell', cell, '-retry-delay', '%ss' % (str(retry_delay)), '-retry-count', str(retry_count), '-log_dir', environment.vtlogroot, '-srv_topo_cache_ttl', cache_ttl, '-conn-timeout-total', timeout_total, '-conn-timeout-per-conn', timeout_per_conn, '-bsonrpc_timeout', '5s', '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-gateway_implementation', vtgate_gateway_flavor().flavor(), '-tablet_types_to_wait', 'MASTER,REPLICA', ] args.extend(vtgate_gateway_flavor().flags(cell=cell)) if protocols_flavor().vtgate_protocol() == 'grpc': args.extend(['-grpc_port', str(self.grpc_port)]) if protocols_flavor().service_map(): args.extend(['-service_map', ','.join(protocols_flavor().service_map())]) if topo_impl: args.extend(['-topo_implementation', topo_impl]) else: args.extend(environment.topo_server().flags()) if extra_args: args.extend(extra_args) self.proc = run_bg(args) if self.secure_port: wait_for_vars('vtgate', self.port, 'SecureConnections') else: wait_for_vars('vtgate', self.port) global vtgate if not vtgate: vtgate = self
def verify_no_endpoint(self, name): """verifies the l2vtgate doesn't have any enpoint of the given name. Args: name: name of the endpoint, in the form: 'keyspace.shard.type'. """ poll_for_vars('l2vtgate', self.port, 'no endpoint named ' + name, timeout=5.0, condition_fn=lambda v: v.get(vtgate_gateway_flavor(). connection_count_vars()). get(name, None) is None)
def wait_for_endpoints(self, name, count, timeout=20.0, var=None): """waits until vtgate gets endpoints. Args: name: name of the endpoint, in the form: 'keyspace.shard.type'. count: how many endpoints to wait for. timeout: how long to wait. var: name of the variable to use. if None, defaults to the gateway's. """ wait_for_vars('vtgate', self.port, var=var or vtgate_gateway_flavor().connection_count_vars(), key=name, value=count, timeout=timeout)
def test_service_switch(self): """tests the service switch from disable -> enable -> disable.""" # make the replica spare utils.run_vtctl( ['ChangeSlaveType', replica_tablet.tablet_alias, 'spare']) utils.wait_for_tablet_type(replica_tablet.tablet_alias, 'spare') # Check UpdateStreamState is disabled. v = utils.get_vars(replica_tablet.port) if v['UpdateStreamState'] != 'Disabled': self.fail( "Update stream service should be 'Disabled' but is '%s'" % v['UpdateStreamState']) start_position = _get_repl_current_position() # Make sure we can't start a new request to vttablet directly. _, stderr = utils.run_vtctl([ 'VtTabletUpdateStream', '-position', start_position, replica_tablet.tablet_alias ], expect_fail=True) self.assertIn('operation not allowed in state NOT_SERVING', stderr) # Make sure we can't start a new request through vtgate. replica_conn = self._get_vtgate_stream_conn() try: for event, resume_timestamp in replica_conn.update_stream( 'test_keyspace', topodata_pb2.REPLICA, event=query_pb2.EventToken(shard='0', position=start_position), shard='0'): self.assertFail('got event(%d): %s' % (resume_timestamp, str(event))) self.assertFail('update_stream terminated with no exception') except dbexceptions.DatabaseError as e: self.assertIn(vtgate_gateway_flavor().no_tablet_found_message(), str(e)) # Go back to replica. utils.run_vtctl( ['ChangeSlaveType', replica_tablet.tablet_alias, 'replica']) utils.wait_for_tablet_type(replica_tablet.tablet_alias, 'replica') # Check UpdateStreamState is enabled. v = utils.get_vars(replica_tablet.port) if v['UpdateStreamState'] != 'Enabled': self.fail("Update stream service should be 'Enabled' but is '%s'" % v['UpdateStreamState'])
def start(self, cell='test_nj', retry_count=2, topo_impl=None, cache_ttl='1s', healthcheck_conn_timeout='2s', extra_args=None, tablets=None, tablet_types_to_wait='MASTER,REPLICA', tablet_filters=None): """Start l2vtgate.""" args = environment.binary_args('l2vtgate') + [ '-port', str(self.port), '-cell', cell, '-retry-count', str(retry_count), '-log_dir', environment.vtlogroot, '-srv_topo_cache_ttl', cache_ttl, '-healthcheck_conn_timeout', healthcheck_conn_timeout, '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-gateway_implementation', vtgate_gateway_flavor().flavor(), ] args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets)) if tablet_types_to_wait: args.extend(['-tablet_types_to_wait', tablet_types_to_wait]) if tablet_filters: args.extend(['-tablet_filters', tablet_filters]) if protocols_flavor().vtgate_protocol() == 'grpc': args.extend(['-grpc_port', str(self.grpc_port)]) if protocols_flavor().service_map(): args.extend(['-service_map', ','.join(protocols_flavor().service_map())]) if topo_impl: args.extend(['-topo_implementation', topo_impl]) else: args.extend(environment.topo_server().flags()) if extra_args: args.extend(extra_args) self.proc = run_bg(args) wait_for_vars('l2vtgate', self.port)
def test_service_switch(self): """tests the service switch from disable -> enable -> disable.""" # make the replica spare utils.run_vtctl(['ChangeSlaveType', replica_tablet.tablet_alias, 'spare']) utils.wait_for_tablet_type(replica_tablet.tablet_alias, 'spare') # Check UpdateStreamState is disabled. v = utils.get_vars(replica_tablet.port) if v['UpdateStreamState'] != 'Disabled': self.fail("Update stream service should be 'Disabled' but is '%s'" % v['UpdateStreamState']) start_position = _get_repl_current_position() # Make sure we can't start a new request to vttablet directly. _, stderr = utils.run_vtctl(['VtTabletUpdateStream', '-position', start_position, replica_tablet.tablet_alias], expect_fail=True) self.assertIn('operation not allowed in state NOT_SERVING', stderr) # Make sure we can't start a new request through vtgate. replica_conn = self._get_vtgate_stream_conn() try: for event, resume_timestamp in replica_conn.update_stream( 'test_keyspace', topodata_pb2.REPLICA, event=query_pb2.EventToken(shard='0', position=start_position), shard='0'): self.assertFail('got event(%d): %s' % (resume_timestamp, str(event))) self.assertFail('update_stream terminated with no exception') except dbexceptions.DatabaseError as e: self.assertIn(vtgate_gateway_flavor().no_tablet_found_message(), str(e)) # Go back to replica. utils.run_vtctl( ['ChangeSlaveType', replica_tablet.tablet_alias, 'replica']) utils.wait_for_tablet_type(replica_tablet.tablet_alias, 'replica') # Check UpdateStreamState is enabled. v = utils.get_vars(replica_tablet.port) if v['UpdateStreamState'] != 'Enabled': self.fail("Update stream service should be 'Enabled' but is '%s'" % v['UpdateStreamState'])
def condition(v): return (v.get(vtgate_gateway_flavor().connection_count_vars()).get( name, None)) is None
def condition(v): return (v.get(vtgate_gateway_flavor().connection_count_vars()) .get(name, None)) is None