def setUpModule():
global vtgateclienttest_process
global vtgateclienttest_port
global vtgateclienttest_grpc_port
try:
environment.topo_server().setup()
vtgateclienttest_port = environment.reserve_ports(1)
args = environment.binary_args('vtgateclienttest') + [
'-log_dir', environment.vtlogroot,
'-port', str(vtgateclienttest_port),
]
if protocols_flavor().vtgate_python_protocol() == 'grpc':
vtgateclienttest_grpc_port = environment.reserve_ports(1)
args.extend(['-grpc_port', str(vtgateclienttest_grpc_port)])
if protocols_flavor().service_map():
args.extend(['-service_map', ','.join(protocols_flavor().service_map())])
vtgateclienttest_process = utils.run_bg(args)
utils.wait_for_vars('vtgateclienttest', vtgateclienttest_port)
except:
tearDownModule()
raise
def vtgate_start(vtport=None, cell='test_nj', retry_delay=1, retry_count=1, topo_impl=None, tablet_bson_encrypted=False, cache_ttl='1s', auth=False, timeout="5s", cert=None, key=None, ca_cert=None):
port = vtport or environment.reserve_ports(1)
secure_port = None
args = [environment.binary_path('vtgate'),
'-port', str(port),
'-cell', cell,
'-retry-delay', '%ss' % (str(retry_delay)),
'-retry-count', str(retry_count),
'-log_dir', environment.vtlogroot,
'-srv_topo_cache_ttl', cache_ttl,
'-timeout', timeout,
] + environment.tabletconn_protocol_flags()
if topo_impl:
args.extend(['-topo_implementation', topo_impl])
else:
args.extend(environment.topo_server_flags())
if tablet_bson_encrypted:
args.append('-tablet-bson-encrypted')
if auth:
args.extend(['-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json')])
if cert:
secure_port = environment.reserve_ports(1)
args.extend(['-secure-port', '%s' % secure_port,
'-cert', cert,
'-key', key])
if ca_cert:
args.extend(['-ca-cert', ca_cert])
sp = run_bg(args)
if cert:
wait_for_vars("vtgate", port, "SecureConnections")
return sp, port, secure_port
else:
wait_for_vars("vtgate", port)
return sp, port
def __init__(self, tablet_uid=None, port=None, mysql_port=None, cell=None,
use_mysqlctld=False):
self.tablet_uid = tablet_uid or (Tablet.default_uid + Tablet.seq)
self.port = port or (environment.reserve_ports(1))
self.mysql_port = mysql_port or (environment.reserve_ports(1))
self.grpc_port = environment.reserve_ports(1)
self.use_mysqlctld = use_mysqlctld
Tablet.seq += 1
if cell:
self.cell = cell
else:
self.cell = tablet_cell_map.get(tablet_uid, 'nj')
self.proc = None
# filled in during init_tablet
self.keyspace = None
self.shard = None
self.index = None
self.tablet_index = None
# utility variables
self.tablet_alias = 'test_%s-%010d' % (self.cell, self.tablet_uid)
self.zk_tablet_path = (
'/zk/test_%s/vt/tablets/%010d' % (self.cell, self.tablet_uid))
def _get_vtworker_cmd(clargs, auto_log=False):
"""Assembles the command that is needed to run a vtworker.
Returns:
cmd - list of cmd arguments, can be passed to any `run`-like functions
port - int with the port number that the vtworker is running with
rpc_port - int with the port number of the RPC interface
"""
port = environment.reserve_ports(1)
rpc_port = port
args = environment.binary_args('vtworker') + [
'-log_dir', environment.vtlogroot,
'-min_healthy_rdonly_endpoints', '1',
'-port', str(port),
'-resolve_ttl', '2s',
'-executefetch_retry_time', '1s',
]
args.extend(environment.topo_server().flags())
args.extend(['-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol()])
if protocols_flavor().service_map():
args.extend(['-service_map',
",".join(protocols_flavor().service_map())])
if protocols_flavor().vtworker_client_protocol() == 'grpc':
rpc_port = environment.reserve_ports(1)
args.extend(['-grpc_port', str(rpc_port)])
if auto_log:
args.append('--stderrthreshold=%s' % get_log_level())
cmd = args + clargs
return cmd, port, rpc_port
def __init__(self, port=None):
"""Creates the Vtgate instance and reserve the ports if necessary."""
self.port = port or environment.reserve_ports(1)
if protocols_flavor().vtgate_protocol() == 'grpc':
self.grpc_port = environment.reserve_ports(1)
self.secure_port = None
self.proc = None
def __init__(self, port=None, mysql_server=False):
"""Creates the Vtgate instance and reserve the ports if necessary."""
self.port = port or environment.reserve_ports(1)
if protocols_flavor().vtgate_protocol() == 'grpc':
self.grpc_port = environment.reserve_ports(1)
self.proc = None
self.mysql_port = None
if mysql_server:
self.mysql_port = environment.reserve_ports(1)
def __init__(self, port=None, cert=None, key=None):
"""Creates the Vtgate instance and reserve the ports if necessary.
"""
self.port = port or environment.reserve_ports(1)
self.secure_port = None
if cert:
self.secure_port = environment.reserve_ports(1)
self.cert = cert
self.key = key
self.proc = None
def __init__(self, port=None, cert=None, key=None):
"""Creates the Vtgate instance and reserve the ports if necessary.
"""
self.port = port or environment.reserve_ports(1)
self.secure_port = None
if cert:
self.secure_port = environment.reserve_ports(1)
self.cert = cert
self.key = key
self.proc = None
def assign_ports(self):
"""Assign ports if not already assigned"""
if self.ports_assigned:
return
from environment import reserve_ports
self.zk_port_base = reserve_ports(3)
self.zkocc_port_base = reserve_ports(3)
self.hostname = socket.gethostname()
self.zk_ports = ':'.join(str(self.zk_port_base + i) for i in range(3))
self.zk_client_port = self.zk_port_base + 2
self.ports_assigned = True
def vtgate_start(vtport=None, cell='test_nj', retry_delay=1, retry_count=1,
topo_impl=None, tablet_bson_encrypted=False, cache_ttl='1s',
auth=False, timeout="5s", cert=None, key=None, ca_cert=None,
socket_file=None, schema=None, extra_args=None):
port = vtport or environment.reserve_ports(1)
secure_port = None
args = environment.binary_args('vtgate') + [
'-port', str(port),
'-cell', cell,
'-retry-delay', '%ss' % (str(retry_delay)),
'-retry-count', str(retry_count),
'-log_dir', environment.vtlogroot,
'-srv_topo_cache_ttl', cache_ttl,
'-timeout', timeout,
] + protocols_flavor().tabletconn_protocol_flags()
if topo_impl:
args.extend(['-topo_implementation', topo_impl])
else:
args.extend(environment.topo_server().flags())
if tablet_bson_encrypted:
args.append('-tablet-bson-encrypted')
if auth:
args.extend(['-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json')])
if cert:
secure_port = environment.reserve_ports(1)
args.extend(['-secure-port', '%s' % secure_port,
'-cert', cert,
'-key', key])
if ca_cert:
args.extend(['-ca_cert', ca_cert])
if socket_file:
args.extend(['-socket_file', socket_file])
if schema:
fname = os.path.join(environment.tmproot, "vtgate_schema.json")
with open(fname, "w") as f:
f.write(schema)
args.extend(['-schema-file', fname])
if extra_args:
args.extend(extra_args)
sp = run_bg(args)
if cert:
wait_for_vars("vtgate", port, "SecureConnections")
return sp, port, secure_port
else:
wait_for_vars("vtgate", port)
return sp, port
def start_vttablet(self, port=None, auth=False, memcache=False, wait_for_state="SERVING", customrules=None, schema_override=None, cert=None, key=None, ca_cert=None, repl_extra_flags={}):
"""
Starts a vttablet process, and returns it.
The process is also saved in self.proc, so it's easy to kill as well.
"""
environment.prog_compile('vtaction')
args = [environment.binary_path('vttablet'),
'-port', '%s' % (port or self.port),
'-tablet-path', self.tablet_alias,
'-log_dir', environment.vtlogroot]
args.extend(environment.topo_server_flags())
args.extend(environment.binlog_player_protocol_flags())
dbconfigs = self._get_db_configs_file(repl_extra_flags)
for key1 in dbconfigs:
for key2 in dbconfigs[key1]:
args.extend(["-db-config-"+key1+"-"+key2, dbconfigs[key1][key2]])
if memcache:
if os.path.exists(environment.vtroot + "/bin/memcached"):
args.extend(["-rowcache-bin", environment.vtroot + "/bin/memcached"])
else:
args.extend(["-rowcache-bin", "memcached"])
memcache_socket = os.path.join(self.tablet_dir, "memcache.sock")
args.extend(["-rowcache-socket", memcache_socket])
args.extend(["-enable-rowcache"])
if auth:
args.extend(['-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json')])
if customrules:
args.extend(['-customrules', customrules])
if schema_override:
args.extend(['-schema-override', schema_override])
if cert:
self.secure_port = environment.reserve_ports(1)
args.extend(['-secure-port', '%s' % self.secure_port,
'-cert', cert,
'-key', key])
if ca_cert:
args.extend(['-ca-cert', ca_cert])
stderr_fd = open(os.path.join(self.tablet_dir, "vttablet.stderr"), "w")
# increment count only the first time
if not self.proc:
Tablet.tablets_running += 1
self.proc = utils.run_bg(args, stderr=stderr_fd)
stderr_fd.close()
# wait for zookeeper PID just to be sure we have it
if environment.topo_server_implementation == 'zookeeper':
utils.run(environment.binary_path('zk')+' wait -e ' + self.zk_pid, stdout=utils.devnull)
# wait for query service to be in the right state
if wait_for_state:
self.wait_for_vttablet_state(wait_for_state, port=port)
return self.proc
def test_standalone(self):
"""Sample test for run_local_database.py as a standalone process.
"""
# launch a backend database based on the provided topology and schema
port = environment.reserve_ports(1)
args = [environment.run_local_database,
'--port', str(port),
'--topology',
'test_keyspace/-80:test_keyspace_0,'
'test_keyspace/80-:test_keyspace_1',
'--schema_dir', os.path.join(environment.vttop, 'test',
'vttest_schema')]
sp = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
config = json.loads(sp.stdout.readline())
# gather the vars for the vtgate process
url = 'http://localhost:%d/debug/vars' % config['port']
f = urllib.urlopen(url)
data = f.read()
f.close()
json_vars = json.loads(data)
self.assertIn('vtgate', json_vars['cmdline'][0])
# and we're done, clean-up
sp.stdin.write('\n')
sp.wait()
def start_vttablet(self, port=None, auth=False, memcache=False, wait_for_state="SERVING", customrules=None, schema_override=None, cert=None, key=None, ca_cert=None, repl_extra_flags={}):
"""
Starts a vttablet process, and returns it.
The process is also saved in self.proc, so it's easy to kill as well.
"""
environment.prog_compile('vtaction')
args = [environment.binary_path('vttablet'),
'-port', '%s' % (port or self.port),
'-tablet-path', self.tablet_alias,
'-log_dir', environment.vtlogroot]
args.extend(environment.topo_server_flags())
args.extend(utils.binlog_player_protocol_flags)
dbconfigs = self._get_db_configs_file(repl_extra_flags)
for key1 in dbconfigs:
for key2 in dbconfigs[key1]:
args.extend(["-db-config-"+key1+"-"+key2, dbconfigs[key1][key2]])
if memcache:
if os.path.exists(environment.vtroot + "/bin/memcached"):
args.extend(["-rowcache-bin", environment.vtroot + "/bin/memcached"])
else:
args.extend(["-rowcache-bin", "memcached"])
memcache_socket = os.path.join(self.tablet_dir, "memcache.sock")
args.extend(["-rowcache-socket", memcache_socket])
args.extend(["-enable-rowcache"])
if auth:
args.extend(['-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json')])
if customrules:
args.extend(['-customrules', customrules])
if schema_override:
args.extend(['-schema-override', schema_override])
if cert:
self.secure_port = environment.reserve_ports(1)
args.extend(['-secure-port', '%s' % self.secure_port,
'-cert', cert,
'-key', key])
if ca_cert:
args.extend(['-ca-cert', ca_cert])
stderr_fd = open(os.path.join(self.tablet_dir, "vttablet.stderr"), "w")
# increment count only the first time
if not self.proc:
Tablet.tablets_running += 1
self.proc = utils.run_bg(args, stderr=stderr_fd)
stderr_fd.close()
# wait for zookeeper PID just to be sure we have it
if environment.topo_server_implementation == 'zookeeper':
utils.run(environment.binary_path('zk')+' wait -e ' + self.zk_pid, stdout=utils.devnull)
# wait for query service to be in the right state
if wait_for_state:
self.wait_for_vttablet_state(wait_for_state, port=port)
return self.proc
def run_vtworker(clargs,
log_level='',
auto_log=False,
expect_fail=False,
**kwargs):
args = [
environment.binary_path('vtworker'), '-log_dir', environment.vtlogroot,
'-port',
str(environment.reserve_ports(1))
]
if auto_log:
if options.verbose == 2:
log_level = 'INFO'
elif options.verbose == 1:
log_level = 'WARNING'
else:
log_level = 'ERROR'
if log_level:
args.append('--stderrthreshold=%s' % log_level)
cmd = args + clargs
if expect_fail:
return run_fail(cmd, **kwargs)
return run(cmd, **kwargs)
def __init__(self, name):
import environment # pylint: disable=g-import-not-at-top
import utils # pylint: disable=g-import-not-at-top
self.port_base = environment.reserve_ports(2)
self.name = name
self.hostname = 'localhost'
self.client_port = self.port_base
self.peer_port = self.port_base + 1
self.client_addr = 'http://%s:%d' % (self.hostname, self.client_port)
self.peer_addr = 'http://%s:%d' % (self.hostname, self.peer_port)
self.api_url = self.client_addr + '/v2'
dirname = 'etcd_' + self.name
self.data_dir = os.path.join(environment.vtdataroot, dirname)
self.proc = utils.run_bg([
'etcd', '-name', self.name,
'-advertise-client-urls', self.client_addr,
'-initial-advertise-peer-urls', self.peer_addr,
'-listen-client-urls', self.client_addr,
'-listen-peer-urls', self.peer_addr,
'-initial-cluster', '%s=%s' % (self.name, self.peer_addr),
'-data-dir', self.data_dir],
stdout=open(os.path.join(
environment.vtlogroot,
dirname + '.stdout'),
'w'),
stderr=open(os.path.join(
environment.vtlogroot,
dirname + '.stderr'),
'w'),)
def setUpClass(cls):
"""Set up two keyspaces: one unsharded, one with two shards."""
topology = vttest_pb2.VTTestTopology()
topology.cells.append('test')
topology.cells.append('test2')
keyspace = topology.keyspaces.add(name='test_keyspace')
keyspace.replica_count = 2
keyspace.rdonly_count = 2
keyspace.shards.add(name='-80')
keyspace.shards.add(name='80-')
keyspace2 = topology.keyspaces.add(name='test_keyspace2')
keyspace2.shards.add(name='0')
keyspace2.replica_count = 2
keyspace2.rdonly_count = 1
cls.driver = environment.create_webdriver()
port = environment.reserve_ports(1)
vttest_environment.base_port = port
environment.reset_mysql_flavor()
cls.db = local_database.LocalDatabase(
topology,
os.path.join(environment.vttop, 'test/vttest_schema'),
False, None,
web_dir=os.path.join(environment.vttop, 'web/vtctld'),
default_schema_dir=os.path.join(
environment.vttop, 'test/vttest_schema/default'),
web_dir2=os.path.join(environment.vttop, 'web/vtctld2/app'))
cls.db.setup()
cls.vtctld_addr = 'http://localhost:%d' % cls.db.config()['port']
utils.pause('Paused test after vtcombo was started.\n'
'For manual testing, connect to vtctld: %s' % cls.vtctld_addr)
def test_standalone(self):
"""Sample test for run_local_database.py as a standalone process.
"""
# launch a backend database based on the provided topology and schema
port = environment.reserve_ports(1)
args = [
environment.run_local_database, '--port',
str(port), '--topology', 'test_keyspace/-80:test_keyspace_0,'
'test_keyspace/80-:test_keyspace_1', '--schema_dir',
os.path.join(environment.vttop, 'test', 'vttest_schema')
]
sp = subprocess.Popen(args,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
config = json.loads(sp.stdout.readline())
# gather the vars for the vtgate process
url = 'http://localhost:%d/debug/vars' % config['port']
f = urllib.urlopen(url)
data = f.read()
f.close()
vars = json.loads(data)
self.assertIn('vtgate', vars['cmdline'][0])
# and we're done, clean-up
sp.stdin.write('\n')
sp.wait()
def _get_vtworker_cmd(clargs, log_level='', auto_log=False):
"""Assembles the command that is needed to run a vtworker.
Returns:
cmd - list of cmd arguments, can be passed to any `run`-like functions
port - int with the port number that the vtworker is running with
"""
port = environment.reserve_ports(1)
args = environment.binary_args('vtworker') + [
'-log_dir', environment.vtlogroot,
'-min_healthy_rdonly_endpoints', '1',
'-port', str(port),
'-resolve_ttl', '2s',
'-executefetch_retry_time', '1s',
]
args.extend(environment.topo_server().flags())
args.extend(protocols_flavor().tablet_manager_protocol_flags())
if auto_log:
if options.verbose == 2:
log_level='INFO'
elif options.verbose == 1:
log_level='WARNING'
else:
log_level='ERROR'
if log_level:
args.append('--stderrthreshold=%s' % log_level)
cmd = args + clargs
return cmd, port
def __init__(self,
tablet_uid=None,
cell=None,
vt_dba_passwd=None,
mysql_port=None):
self.tablet_uid = tablet_uid or (Vtbackup.default_uid + Vtbackup.seq)
self.mysql_port = mysql_port or (environment.reserve_ports(1))
self.vt_dba_passwd = vt_dba_passwd
Vtbackup.seq += 1
if cell:
self.cell = cell
else:
self.cell = tablet_cell_map.get(tablet_uid, 'nj')
self.proc = None
# filled in during init_mysql
self.mysqlctld_process = None
# filled in during init_tablet
self.keyspace = None
self.shard = None
self.index = None
self.tablet_index = None
self.dbname = None
# default to false
self.external_mysql = False
# utility variables
self.tablet_alias = 'test_%s-%010d' % (self.cell, self.tablet_uid)
def run_vtworker(clargs,
log_level='',
auto_log=False,
expect_fail=False,
**kwargs):
args = environment.binary_args('vtworker') + [
'-log_dir', environment.vtlogroot, '-port',
str(environment.reserve_ports(1))
]
args.extend(environment.topo_server().flags())
args.extend(protocols_flavor().tablet_manager_protocol_flags())
if auto_log:
if options.verbose == 2:
log_level = 'INFO'
elif options.verbose == 1:
log_level = 'WARNING'
else:
log_level = 'ERROR'
if log_level:
args.append('--stderrthreshold=%s' % log_level)
cmd = args + clargs
if expect_fail:
return run_fail(cmd, **kwargs)
return run(cmd, **kwargs)
def __init__(self, name):
import environment
import utils
self.port_base = environment.reserve_ports(2)
self.name = name
self.hostname = 'localhost'
self.client_port = self.port_base
self.peer_port = self.port_base + 1
self.client_addr = 'http://%s:%u' % (self.hostname, self.client_port)
self.peer_addr = 'http://%s:%u' % (self.hostname, self.peer_port)
self.api_url = self.client_addr + '/v2'
dirname = 'etcd_' + self.name
self.data_dir = os.path.join(environment.vtdataroot, dirname)
self.proc = utils.run_bg(
[
'etcd', '-name', self.name, '-advertise-client-urls',
self.client_addr, '-initial-advertise-peer-urls',
self.peer_addr, '-listen-client-urls', self.client_addr,
'-listen-peer-urls', self.peer_addr, '-initial-cluster',
'%s=%s' %
(self.name, self.peer_addr), '-data-dir', self.data_dir
],
stdout=open(
os.path.join(environment.vtlogroot, dirname + '.stdout'), 'w'),
stderr=open(
os.path.join(environment.vtlogroot, dirname + '.stderr'), 'w'),
)
def __init__(self, name):
import environment
import utils
self.port_base = environment.reserve_ports(2)
self.name = name
self.hostname = 'localhost'
self.client_port = self.port_base
self.peer_port = self.port_base + 1
self.client_addr = '%s:%u' % (self.hostname, self.client_port)
self.peer_addr = '%s:%u' % (self.hostname, self.peer_port)
self.api_url = 'http://%s/v2' % (self.client_addr)
dirname = 'etcd_' + self.name
self.data_dir = os.path.join(environment.vtdataroot, dirname)
self.proc = utils.run_bg([
'etcd', '-name', self.name, '-addr',
self.client_addr, '-peer-addr',
self.peer_addr, '-data-dir', self.data_dir],
stdout=open(os.path.join(
environment.vtlogroot,
dirname + '.stdout'),
'w'),
stderr=open(os.path.join(
environment.vtlogroot,
dirname + '.stderr'),
'w'),)
def _get_vtworker_cmd(clargs, auto_log=False):
"""Assembles the command that is needed to run a vtworker.
Args:
clargs: Command line arguments passed to vtworker.
auto_log: If true, set --stderrthreshold according to the test log level.
Returns:
cmd - list of cmd arguments, can be passed to any `run`-like functions
port - int with the port number that the vtworker is running with
rpc_port - int with the port number of the RPC interface
"""
port = environment.reserve_ports(1)
rpc_port = port
args = environment.binary_args('vtworker') + [
'-log_dir',
environment.vtlogroot,
'-min_healthy_rdonly_endpoints',
'1',
'-port',
str(port),
# use a long resolve TTL because of potential race conditions with doing
# an EmergencyReparent and resolving the master (as EmergencyReparent
# will delete the old master before updating the shard record with the
# new master)
'-resolve_ttl',
'10s',
'-executefetch_retry_time',
'1s',
'-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol(),
'-tablet_protocol',
protocols_flavor().tabletconn_protocol(),
]
args.extend(environment.topo_server().flags())
if protocols_flavor().service_map():
args.extend(
['-service_map', ','.join(protocols_flavor().service_map())])
if protocols_flavor().vtworker_client_protocol() == 'grpc':
rpc_port = environment.reserve_ports(1)
args.extend(['-grpc_port', str(rpc_port)])
if auto_log:
args.append('--stderrthreshold=%s' % get_log_level())
cmd = args + clargs
return cmd, port, rpc_port
def _get_vtworker_cmd(clargs, auto_log=False):
"""Assembles the command that is needed to run a vtworker.
Args:
clargs: Command line arguments passed to vtworker.
auto_log: If true, set --stderrthreshold according to the test log level.
Returns:
cmd - list of cmd arguments, can be passed to any `run`-like functions
port - int with the port number that the vtworker is running with
rpc_port - int with the port number of the RPC interface
"""
port = environment.reserve_ports(1)
rpc_port = port
args = environment.binary_args("vtworker") + [
"-log_dir",
environment.vtlogroot,
"-min_healthy_rdonly_endpoints",
"1",
"-port",
str(port),
# use a long resolve TTL because of potential race conditions with doing
# an EmergencyReparent and resolving the master (as EmergencyReparent
# will delete the old master before updating the shard record with the
# new master)
"-resolve_ttl",
"10s",
"-executefetch_retry_time",
"1s",
"-tablet_manager_protocol",
protocols_flavor().tablet_manager_protocol(),
"-tablet_protocol",
protocols_flavor().tabletconn_protocol(),
]
args.extend(environment.topo_server().flags())
if protocols_flavor().service_map():
args.extend(["-service_map", ",".join(protocols_flavor().service_map())])
if protocols_flavor().vtworker_client_protocol() == "grpc":
rpc_port = environment.reserve_ports(1)
args.extend(["-grpc_port", str(rpc_port)])
if auto_log:
args.append("--stderrthreshold=%s" % get_log_level())
cmd = args + clargs
return cmd, port, rpc_port
def setup(self, add_bad_host=False):
from environment import reserve_ports, run, binary_args, vtlogroot, tmproot
self.zk_port_base = reserve_ports(3)
self.zkocc_port_base = reserve_ports(3)
self.hostname = socket.gethostname()
self.zk_ports = ':'.join(str(self.zk_port_base + i) for i in range(3))
self.zk_client_port = self.zk_port_base + 2
run(
binary_args('zkctl') + [
'-log_dir', vtlogroot, '-zk.cfg',
'1@%s:%s' % (self.hostname, self.zk_ports), 'init'
])
config = tmproot + '/test-zk-client-conf.json'
with open(config, 'w') as f:
ca_server = 'localhost:%u' % (self.zk_client_port)
if add_bad_host:
ca_server += ',does.not.exists:1234'
zk_cell_mapping = {
'test_nj':
'localhost:%u' % (self.zk_client_port),
'test_ny':
'localhost:%u' % (self.zk_client_port),
'test_ca':
ca_server,
'global':
'localhost:%u' % (self.zk_client_port),
'test_nj:_zkocc':
'localhost:%u,localhost:%u,localhost:%u' %
tuple(self.zkocc_port_base + i for i in range(3)),
'test_ny:_zkocc':
'localhost:%u' % (self.zkocc_port_base),
'test_ca:_zkocc':
'localhost:%u' % (self.zkocc_port_base),
'global:_zkocc':
'localhost:%u' % (self.zkocc_port_base),
}
json.dump(zk_cell_mapping, f)
os.environ['ZK_CLIENT_CONFIG'] = config
logging.debug('Using ZK_CLIENT_CONFIG=%s', str(config))
run(binary_args('zk') + ['touch', '-p', '/zk/test_nj/vt'])
run(binary_args('zk') + ['touch', '-p', '/zk/test_ny/vt'])
run(binary_args('zk') + ['touch', '-p', '/zk/test_ca/vt'])
def _get_vtworker_cmd(clargs, log_level='', auto_log=False):
"""Assembles the command that is needed to run a vtworker.
Returns:
cmd - list of cmd arguments, can be passed to any `run`-like functions
port - int with the port number that the vtworker is running with
"""
port = environment.reserve_ports(1)
rpc_port = environment.reserve_ports(1)
args = environment.binary_args('vtworker') + [
'-log_dir',
environment.vtlogroot,
'-min_healthy_rdonly_endpoints',
'1',
'-port',
str(port),
'-resolve_ttl',
'2s',
'-executefetch_retry_time',
'1s',
]
args.extend(environment.topo_server().flags())
args.extend([
'-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol()
])
if protocols_flavor().service_map():
args.extend(
['-service_map', ",".join(protocols_flavor().service_map())])
if protocols_flavor().vtworker_client_protocol() == 'grpc':
args.extend(['-grpc_port', str(rpc_port)])
if auto_log:
if options.verbose == 2:
log_level = 'INFO'
elif options.verbose == 1:
log_level = 'WARNING'
else:
log_level = 'ERROR'
if log_level:
args.append('--stderrthreshold=%s' % log_level)
cmd = args + clargs
return cmd, port, rpc_port
def setUpClass(cls):
cls.driver = environment.create_webdriver()
port = environment.reserve_ports(1)
keytar_folder = os.path.join(environment.vttop, 'test/cluster/keytar')
cls.flask_process = subprocess.Popen(
[os.path.join(keytar_folder, 'keytar.py'),
'--config_file=%s' % os.path.join(keytar_folder, 'test_config.yaml'),
'--port=%d' % port, '--password=foo'],
preexec_fn=os.setsid)
cls.flask_addr = 'http://localhost:%d' % port
def vtgate_start(vtport=None, cell='test_nj', retry_delay=1, retry_count=1,
topo_impl=None, tablet_bson_encrypted=False, cache_ttl='1s',
auth=False, timeout="5s", cert=None, key=None, ca_cert=None,
socket_file=None, extra_args=None):
port = vtport or environment.reserve_ports(1)
secure_port = None
args = environment.binary_args('vtgate') + [
'-port', str(port),
'-cell', cell,
'-retry-delay', '%ss' % (str(retry_delay)),
'-retry-count', str(retry_count),
'-log_dir', environment.vtlogroot,
'-srv_topo_cache_ttl', cache_ttl,
'-timeout', timeout,
] + environment.tabletconn_protocol_flags()
if topo_impl:
args.extend(['-topo_implementation', topo_impl])
else:
args.extend(environment.topo_server_flags())
if tablet_bson_encrypted:
args.append('-tablet-bson-encrypted')
if auth:
args.extend(['-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json')])
if cert:
secure_port = environment.reserve_ports(1)
args.extend(['-secure-port', '%s' % secure_port,
'-cert', cert,
'-key', key])
if ca_cert:
args.extend(['-ca_cert', ca_cert])
if socket_file:
args.extend(['-socket_file', socket_file])
if extra_args:
args.extend(extra_args)
sp = run_bg(args)
if cert:
wait_for_vars("vtgate", port, "SecureConnections")
return sp, port, secure_port
else:
wait_for_vars("vtgate", port)
return sp, port
def __init__(self, tablet_uid=None, port=None, mysql_port=None, cell=None):
self.tablet_uid = tablet_uid or (Tablet.default_uid + Tablet.seq)
self.port = port or (environment.reserve_ports(1))
self.mysql_port = mysql_port or (environment.reserve_ports(1))
Tablet.seq += 1
if cell:
self.cell = cell
else:
self.cell = tablet_cell_map.get(tablet_uid, 'nj')
self.proc = None
# filled in during init_tablet
self.keyspace = None
self.shard = None
# utility variables
self.tablet_alias = 'test_%s-%010d' % (self.cell, self.tablet_uid)
self.zk_tablet_path = '/zk/test_%s/vt/tablets/%010d' % (self.cell, self.tablet_uid)
self.zk_pid = self.zk_tablet_path + '/pid'
def __init__(self, tablet_uid=None, port=None, mysql_port=None, cell=None):
self.tablet_uid = tablet_uid or (Tablet.default_uid + Tablet.seq)
self.port = port or (environment.reserve_ports(1))
self.mysql_port = mysql_port or (environment.reserve_ports(1))
Tablet.seq += 1
if cell:
self.cell = cell
else:
self.cell = tablet_cell_map.get(tablet_uid, "nj")
self.proc = None
# filled in during init_tablet
self.keyspace = None
self.shard = None
# utility variables
self.tablet_alias = "test_%s-%010d" % (self.cell, self.tablet_uid)
self.zk_tablet_path = "/zk/test_%s/vt/tablets/%010d" % (self.cell, self.tablet_uid)
self.zk_pid = self.zk_tablet_path + "/pid"
def setUpClass(cls):
"""Set up two keyspaces: one unsharded, one with two shards."""
topology = vttest_pb2.VTTestTopology()
topology.cells.append('test')
topology.cells.append('test2')
keyspace = topology.keyspaces.add(name='test_keyspace')
keyspace.replica_count = 2
keyspace.rdonly_count = 2
keyspace.shards.add(name='-80')
keyspace.shards.add(name='80-')
keyspace2 = topology.keyspaces.add(name='test_keyspace2')
keyspace2.shards.add(name='0')
keyspace2.replica_count = 2
keyspace2.rdonly_count = 1
if os.environ.get('CI') == 'true' and os.environ.get(
'TRAVIS') == 'true':
username = os.environ['SAUCE_USERNAME']
access_key = os.environ['SAUCE_ACCESS_KEY']
capabilities = {}
capabilities['tunnel-identifier'] = os.environ['TRAVIS_JOB_NUMBER']
capabilities['build'] = os.environ['TRAVIS_BUILD_NUMBER']
capabilities['platform'] = 'Linux'
capabilities['browserName'] = 'chrome'
hub_url = '%s:%s@localhost:4445' % (username, access_key)
cls.driver = webdriver.Remote(desired_capabilities=capabilities,
command_executor='http://%s/wd/hub' %
hub_url)
else:
os.environ['webdriver.chrome.driver'] = os.path.join(
environment.vtroot, 'dist')
# Only testing against Chrome for now
cls.driver = webdriver.Chrome()
cls.driver.set_window_position(0, 0)
cls.driver.set_window_size(1920, 1280)
port = environment.reserve_ports(1)
vttest_environment.base_port = port
mysql_flavor.set_mysql_flavor(None)
cls.db = local_database.LocalDatabase(
topology,
os.path.join(environment.vttop, 'test/vttest_schema'),
False,
None,
web_dir=os.path.join(environment.vttop, 'web/vtctld'),
default_schema_dir=os.path.join(environment.vttop,
'test/vttest_schema/default'),
web_dir2=os.path.join(environment.vttop, 'web/vtctld2/app'))
cls.db.setup()
cls.vtctld_addr = 'http://localhost:%d' % cls.db.config()['port']
utils.pause('Paused test after vtcombo was started.\n'
'For manual testing, connect to vtctld: %s' %
cls.vtctld_addr)
def setUpClass(cls):
cls.driver = environment.create_webdriver()
port = environment.reserve_ports(1)
keytar_folder = os.path.join(environment.vttop, 'test/cluster/keytar')
cls.flask_process = subprocess.Popen([
os.path.join(keytar_folder, 'keytar.py'),
'--config_file=%s' %
os.path.join(keytar_folder, 'test_config.yaml'),
'--port=%d' % port, '--password=foo'
],
preexec_fn=os.setsid)
cls.flask_addr = 'http://localhost:%d' % port
def setUpClass(cls):
"""Set up two keyspaces: one unsharded, one with two shards."""
topology = vttest_pb2.VTTestTopology()
topology.cells.append("test")
topology.cells.append("test2")
keyspace = topology.keyspaces.add(name="test_keyspace")
keyspace.replica_count = 2
keyspace.rdonly_count = 2
keyspace.shards.add(name="-80")
keyspace.shards.add(name="80-")
keyspace2 = topology.keyspaces.add(name="test_keyspace2")
keyspace2.shards.add(name="0")
keyspace2.replica_count = 2
keyspace2.rdonly_count = 1
if os.environ.get("CI") == "true" and os.environ.get("TRAVIS") == "true":
username = os.environ["SAUCE_USERNAME"]
access_key = os.environ["SAUCE_ACCESS_KEY"]
capabilities = {}
capabilities["tunnel-identifier"] = os.environ["TRAVIS_JOB_NUMBER"]
capabilities["build"] = os.environ["TRAVIS_BUILD_NUMBER"]
capabilities["platform"] = "Linux"
capabilities["browserName"] = "chrome"
hub_url = "%s:%s@localhost:4445" % (username, access_key)
cls.driver = webdriver.Remote(
desired_capabilities=capabilities, command_executor="http://%s/wd/hub" % hub_url
)
else:
os.environ["webdriver.chrome.driver"] = os.path.join(environment.vtroot, "dist")
# Only testing against Chrome for now
cls.driver = webdriver.Chrome()
cls.driver.set_window_position(0, 0)
cls.driver.set_window_size(1920, 1280)
port = environment.reserve_ports(1)
vttest_environment.base_port = port
mysql_flavor.set_mysql_flavor(None)
cls.db = local_database.LocalDatabase(
topology,
os.path.join(environment.vttop, "test/vttest_schema"),
False,
None,
web_dir=os.path.join(environment.vttop, "web/vtctld"),
default_schema_dir=os.path.join(environment.vttop, "test/vttest_schema/default"),
web_dir2=os.path.join(environment.vttop, "web/vtctld2/app"),
)
cls.db.setup()
cls.vtctld_addr = "http://localhost:%d" % cls.db.config()["port"]
utils.pause(
"Paused test after vtcombo was started.\n" "For manual testing, connect to vtctld: %s" % cls.vtctld_addr
)
def _get_vtworker_cmd(clargs, auto_log=False):
"""Assembles the command that is needed to run a vtworker.
Args:
clargs: Command line arguments passed to vtworker.
auto_log: If true, set --stderrthreshold according to the test log level.
Returns:
cmd - list of cmd arguments, can be passed to any `run`-like functions
port - int with the port number that the vtworker is running with
rpc_port - int with the port number of the RPC interface
"""
port = environment.reserve_ports(1)
rpc_port = port
args = environment.binary_args('vtworker') + [
'-log_dir',
environment.vtlogroot,
'-port',
str(port),
'-executefetch_retry_time',
'1s',
'-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol(),
'-tablet_protocol',
protocols_flavor().tabletconn_protocol(),
]
args.extend(environment.topo_server().flags())
if protocols_flavor().service_map():
args.extend(
['-service_map', ','.join(protocols_flavor().service_map())])
if protocols_flavor().vtworker_client_protocol() == 'grpc':
rpc_port = environment.reserve_ports(1)
args.extend(['-grpc_port', str(rpc_port)])
if auto_log:
args.append('--stderrthreshold=%s' % get_log_level())
cmd = args + clargs
return cmd, port, rpc_port
def assign_ports(self):
"""Assign ports if not already assigned"""
if self.ports_assigned:
return
from environment import reserve_ports
self.zk_port_base = reserve_ports(3)
self.hostname = socket.getfqdn()
self.zk_ports = ':'.join(str(self.zk_port_base + i) for i in range(3))
self.zk_client_port = self.zk_port_base + 2
self.ports_assigned = True
def assign_ports(self):
"""Assign ports if not already assigned."""
if self.ports_assigned:
return
from environment import reserve_ports # pylint: disable=g-import-not-at-top
import utils # pylint: disable=g-import-not-at-top
self.zk_port_base = reserve_ports(3)
self.hostname = utils.hostname
self.zk_ports = ':'.join(str(self.zk_port_base + i) for i in range(3))
self.zk_client_port = self.zk_port_base + 2
self.ports_assigned = True
def assign_ports(self):
"""Assign ports if not already assigned."""
if self.ports_assigned:
return
from environment import reserve_ports # pylint: disable=g-import-not-at-top
import utils # pylint: disable=g-import-not-at-top
self.zk_port_base = reserve_ports(3)
self.hostname = utils.hostname
self.zk_ports = ':'.join(str(self.zk_port_base + i) for i in range(3))
self.addr = 'localhost:%d' % (self.zk_port_base + 2)
self.ports_assigned = True
def _get_vtworker_cmd(clargs, auto_log=False):
"""Assembles the command that is needed to run a vtworker.
Returns:
cmd - list of cmd arguments, can be passed to any `run`-like functions
port - int with the port number that the vtworker is running with
rpc_port - int with the port number of the RPC interface
"""
port = environment.reserve_ports(1)
rpc_port = port
args = environment.binary_args('vtworker') + [
'-log_dir', environment.vtlogroot,
'-min_healthy_rdonly_endpoints', '1',
'-port', str(port),
# use a long resolve TTL because of potential race conditions with doing
# an EmergencyReparent and resolving the master (as EmergencyReparent
# will delete the old master before updating the shard record with the
# new master)
'-resolve_ttl', '10s',
'-executefetch_retry_time', '1s',
'-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol(),
'-tablet_protocol', protocols_flavor().tabletconn_protocol(),
]
args.extend(environment.topo_server().flags())
if protocols_flavor().service_map():
args.extend(['-service_map',
','.join(protocols_flavor().service_map())])
if protocols_flavor().vtworker_client_protocol() == 'grpc':
rpc_port = environment.reserve_ports(1)
args.extend(['-grpc_port', str(rpc_port)])
if auto_log:
args.append('--stderrthreshold=%s' % get_log_level())
cmd = args + clargs
return cmd, port, rpc_port
def setUpModule():
global vtgateclienttest_process
global vtgateclienttest_port
global vtgateclienttest_grpc_port
environment.topo_server().setup()
vtgateclienttest_port = environment.reserve_ports(1)
args = environment.binary_args('vtgateclienttest') + [
'-log_dir',
environment.vtlogroot,
'-port',
str(vtgateclienttest_port),
]
if protocols_flavor().vtgate_python_protocol() == 'grpc':
vtgateclienttest_grpc_port = environment.reserve_ports(1)
args.extend(['-grpc_port', str(vtgateclienttest_grpc_port)])
if protocols_flavor().service_map():
args.extend(
['-service_map', ','.join(protocols_flavor().service_map())])
vtgateclienttest_process = utils.run_bg(args)
utils.wait_for_vars('vtgateclienttest', vtgateclienttest_port)
def _test_standalone(self, use_vtcombo):
"""Sample test for run_local_database.py as a standalone process."""
# launch a backend database based on the provided topology and schema
port = environment.reserve_ports(1)
args = [
environment.run_local_database,
'--port',
str(port),
'--topology',
'test_keyspace/-80:test_keyspace_0,'
'test_keyspace/80-:test_keyspace_1',
'--schema_dir',
os.path.join(environment.vttop, 'test', 'vttest_schema'),
]
if use_vtcombo:
args.append('--use_vtcombo')
sp = subprocess.Popen(args,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
config = json.loads(sp.stdout.readline())
# gather the vars for the vtgate process
url = 'http://localhost:%d/debug/vars' % config['port']
f = urllib.urlopen(url)
data = f.read()
f.close()
json_vars = json.loads(data)
process_name = 'vtgate'
if use_vtcombo:
process_name = 'vtcombo'
self.assertIn(process_name, json_vars['cmdline'][0])
# to test vtcombo:
# ./vttest_sample_test.py -v -d
# go install && vtcombo -port 15010 -grpc_port 15011 -service_map grpc-vtgateservice -topology test_keyspace/-80:test_keyspace_0,test_keyspace/80-:test_keyspace_1 -mycnf_server_id 1 -mycnf_socket_file $VTDATAROOT/vttest*/vt_0000000001/mysql.sock -db-config-dba-uname vt_dba -db-config-dba-charset utf8 -db-config-app-uname vt_app -db-config-app-charset utf8 -alsologtostderr
# vtctl -vtgate_protocol grpc VtGateExecuteShards -server localhost:15011 -keyspace test_keyspace -shards -80 -tablet_type master "select 1 from dual"
# vtctl -vtgate_protocol grpc VtGateExecuteKeyspaceIds -server localhost:15011 -keyspace test_keyspace -keyspace_ids 20 -tablet_type master "show tables"
if use_vtcombo:
utils.pause('good time to test vtcombo with database running')
else:
utils.pause('good time to test vtgate with database running')
# and we're done, clean-up
sp.stdin.write('\n')
sp.wait()
def run_automation_server(auto_log=False):
"""Starts a background automation_server process.
Returns:
rpc_port - int with the port number of the RPC interface
"""
rpc_port = environment.reserve_ports(1)
args = environment.binary_args('automation_server') + [
'-log_dir', environment.vtlogroot,
'-port', str(rpc_port),
'-vtctl_client_protocol', protocols_flavor().vtctl_client_protocol(),
'-vtworker_client_protocol', protocols_flavor().vtworker_client_protocol(),
]
if auto_log:
args.append('--stderrthreshold=%s' % get_log_level())
return run_bg(args), rpc_port
def vtgate_start(cell='test_nj', retry_delay=1, retry_count=1, topo_impl=None, tablet_bson_encrypted=False):
port = environment.reserve_ports(1)
args = [environment.binary_path('vtgate'),
'-port', str(port),
'-cell', cell,
'-retry-delay', '%ss' % (str(retry_delay)),
'-retry-count', str(retry_count),
'-log_dir', environment.vtlogroot,
] + environment.vtgate_protocol_flags()
if topo_impl:
args.extend(['-topo_implementation', topo_impl])
else:
args.extend(environment.topo_server_flags())
if tablet_bson_encrypted:
args.append('-tablet-bson-encrypted')
sp = run_bg(args)
wait_for_vars("vtgate", port)
return sp, port
def __init__(self, name):
import environment # pylint: disable=g-import-not-at-top
self.port_base = environment.reserve_ports(2)
self.name = name
self.hostname = 'localhost'
self.client_port = self.port_base
self.peer_port = self.port_base + 1
self.client_addr = 'http://%s:%d' % (self.hostname, self.client_port)
self.peer_addr = 'http://%s:%d' % (self.hostname, self.peer_port)
self.api_url = self.client_addr + '/v2'
dirname = 'etcd_' + self.name
self.data_dir = os.path.join(environment.vtdataroot, dirname)
self.log_base = os.path.join(environment.vtlogroot, dirname)
self.start()
def run_automation_server(auto_log=False):
"""Starts a background automation_server process.
Returns:
rpc_port - int with the port number of the RPC interface
"""
rpc_port = environment.reserve_ports(1)
args = environment.binary_args('automation_server') + [
'-log_dir', environment.vtlogroot,
'-port', str(rpc_port),
'-vtctl_client_protocol', protocols_flavor().vtctl_client_protocol(),
'-vtworker_client_protocol',
protocols_flavor().vtworker_client_protocol(),
]
if auto_log:
args.append('--stderrthreshold=%s' % get_log_level())
return run_bg(args), rpc_port
def __init__(self, name):
import environment # pylint: disable=g-import-not-at-top
self.port_base = environment.reserve_ports(2)
self.name = name
self.hostname = 'localhost'
self.client_port = self.port_base
self.peer_port = self.port_base + 1
self.client_addr = 'http://%s:%d' % (self.hostname, self.client_port)
self.peer_addr = 'http://%s:%d' % (self.hostname, self.peer_port)
self.api_url = self.client_addr + '/v2'
dirname = 'etcd_' + self.name
self.data_dir = os.path.join(environment.vtdataroot, dirname)
self.log_base = os.path.join(environment.vtlogroot, dirname)
self.start()
def setUpClass(cls):
"""Set up two keyspaces: one unsharded, one with two shards."""
if os.environ.get('CI') == 'true' and os.environ.get('TRAVIS') == 'true':
username = os.environ['SAUCE_USERNAME']
access_key = os.environ['SAUCE_ACCESS_KEY']
capabilities = {}
capabilities['tunnel-identifier'] = os.environ['TRAVIS_JOB_NUMBER']
capabilities['build'] = os.environ['TRAVIS_BUILD_NUMBER']
capabilities['platform'] = 'Linux'
capabilities['browserName'] = 'chrome'
hub_url = '%s:%s@localhost:4445' % (username, access_key)
cls.driver = webdriver.Remote(
desired_capabilities=capabilities,
command_executor='http://%s/wd/hub' % hub_url)
else:
os.environ['webdriver.chrome.driver'] = os.path.join(
os.environ['VTROOT'], 'dist')
# Only testing against Chrome for now
cls.driver = webdriver.Chrome()
topology = vttest_pb2.VTTestTopology()
topology.cells.append('test')
keyspace = topology.keyspaces.add(name='test_keyspace')
keyspace.replica_count = 2
keyspace.rdonly_count = 2
keyspace.shards.add(name='-80')
keyspace.shards.add(name='80-')
keyspace2 = topology.keyspaces.add(name='test_keyspace2')
keyspace2.shards.add(name='0')
keyspace2.replica_count = 2
keyspace2.rdonly_count = 1
port = environment.reserve_ports(1)
vttest_environment.base_port = port
mysql_flavor.set_mysql_flavor(None)
cls.db = local_database.LocalDatabase(
topology, '', False, None,
os.path.join(os.environ['VTTOP'], 'web/vtctld2/dist'),
os.path.join(os.environ['VTTOP'], 'test/vttest_schema/default'))
cls.db.setup()
cls.vtctld_addr = 'http://localhost:%d' % cls.db.config()['port']
utils.pause('Paused test after vtcombo was started.\n'
'For manual testing, connect to vtctld: %s' % cls.vtctld_addr)
def run_vtworker(clargs, log_level='', auto_log=False, expect_fail=False, **kwargs):
args = [environment.binary_path('vtworker'),
'-log_dir', environment.vtlogroot,
'-port', str(environment.reserve_ports(1))]
if auto_log:
if options.verbose == 2:
log_level='INFO'
elif options.verbose == 1:
log_level='WARNING'
else:
log_level='ERROR'
if log_level:
args.append('--stderrthreshold=%s' % log_level)
cmd = args + clargs
if expect_fail:
return run_fail(cmd, **kwargs)
return run(cmd, **kwargs)
def _test_standalone(self, use_vtcombo):
"""Sample test for run_local_database.py as a standalone process."""
# launch a backend database based on the provided topology and schema
port = environment.reserve_ports(1)
args = [environment.run_local_database,
'--port', str(port),
'--topology',
'test_keyspace/-80:test_keyspace_0,'
'test_keyspace/80-:test_keyspace_1',
'--schema_dir', os.path.join(environment.vttop, 'test',
'vttest_schema'),
]
if use_vtcombo:
args.append('--use_vtcombo')
sp = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
config = json.loads(sp.stdout.readline())
# gather the vars for the vtgate process
url = 'http://localhost:%d/debug/vars' % config['port']
f = urllib.urlopen(url)
data = f.read()
f.close()
json_vars = json.loads(data)
process_name = 'vtgate'
if use_vtcombo:
process_name = 'vtcombo'
self.assertIn(process_name, json_vars['cmdline'][0])
# to test vtcombo:
# ./vttest_sample_test.py -v -d
# go install && vtcombo -port 15010 -grpc_port 15011 -service_map grpc-vtgateservice -topology test_keyspace/-80:test_keyspace_0,test_keyspace/80-:test_keyspace_1 -mycnf_server_id 1 -mycnf_socket_file $VTDATAROOT/vttest*/vt_0000000001/mysql.sock -db-config-dba-uname vt_dba -db-config-dba-charset utf8 -db-config-app-uname vt_app -db-config-app-charset utf8 -alsologtostderr
# vtctl -vtgate_protocol grpc VtGateExecuteShards -server localhost:15011 -keyspace test_keyspace -shards -80 -tablet_type master "select 1 from dual"
# vtctl -vtgate_protocol grpc VtGateExecuteKeyspaceIds -server localhost:15011 -keyspace test_keyspace -keyspace_ids 20 -tablet_type master "show tables"
if use_vtcombo:
utils.pause('good time to test vtcombo with database running')
else:
utils.pause('good time to test vtgate with database running')
# and we're done, clean-up
sp.stdin.write('\n')
sp.wait()
def run_vtworker(clargs, log_level='', auto_log=False, expect_fail=False, **kwargs):
args = environment.binary_args('vtworker') + [
'-log_dir', environment.vtlogroot,
'-port', str(environment.reserve_ports(1))]
args.extend(environment.topo_server_flags())
args.extend(environment.tablet_manager_protocol_flags())
if auto_log:
if options.verbose == 2:
log_level='INFO'
elif options.verbose == 1:
log_level='WARNING'
else:
log_level='ERROR'
if log_level:
args.append('--stderrthreshold=%s' % log_level)
cmd = args + clargs
if expect_fail:
return run_fail(cmd, **kwargs)
return run(cmd, **kwargs)
def setup(self):
import environment # pylint: disable=g-import-not-at-top
import utils # pylint: disable=g-import-not-at-top
self.port_base = environment.reserve_ports(5)
self.server_addr = 'localhost:%d' % (self.port_base + 1)
# Write our config file.
self.config_file = os.path.join(environment.vtdataroot, 'consul.json')
config = {
'ports': {
'dns': self.port_base,
'http': self.port_base + 1,
'rpc': self.port_base + 2,
'serf_lan': self.port_base + 3,
'serf_wan': self.port_base + 4,
},
}
with open(self.config_file, 'w') as fd:
fd.write(json.dumps(config))
log_base = os.path.join(environment.vtlogroot, 'consul')
self.proc = utils.run_bg([
'consul', 'agent',
'-dev',
'-config-file', self.config_file],
stdout=open(log_base + '.stdout', 'a'),
stderr=open(log_base + '.stderr', 'a'))
# Wait until the daemon is ready.
utils.curl(
'http://' + self.server_addr + '/v1/kv/?keys', retry_timeout=10)
# Create the cell configurations using 'vtctl AddCellInfo'
for cell in ['test_nj', 'test_ny', 'test_ca']:
utils.run_vtctl_vtctl(['AddCellInfo',
'-root', cell,
'-server_address', self.server_addr,
cell])
def setup(self):
import environment # pylint: disable=g-import-not-at-top
import utils # pylint: disable=g-import-not-at-top
self.port_base = environment.reserve_ports(5)
self.server_addr = 'localhost:%d' % (self.port_base + 1)
# Write our config file.
self.config_file = os.path.join(environment.vtdataroot, 'consul.json')
config = {
'ports': {
'dns': self.port_base,
'http': self.port_base + 1,
'rpc': self.port_base + 2,
'serf_lan': self.port_base + 3,
'serf_wan': self.port_base + 4,
},
}
with open(self.config_file, 'w') as fd:
fd.write(json.dumps(config))
log_base = os.path.join(environment.vtlogroot, 'consul')
self.proc = utils.run_bg([
'consul', 'agent',
'-dev',
'-config-file', self.config_file],
stdout=open(log_base + '.stdout', 'a'),
stderr=open(log_base + '.stderr', 'a'))
# Wait until the daemon is ready.
utils.curl(
'http://' + self.server_addr + '/v1/kv/?keys', retry_timeout=10)
# Create the cell configurations using 'vtctl AddCellInfo'
for cell in ['test_nj', 'test_ny', 'test_ca']:
utils.run_vtctl_vtctl(['AddCellInfo',
'-root', cell,
'-server_address', self.server_addr,
cell])
def run_automation_server(auto_log=False):
"""Starts a background automation_server process.
Args:
auto_log: True to log.
Returns:
rpc_port - int with the port number of the RPC interface
"""
rpc_port = environment.reserve_ports(1)
args = environment.binary_args("automation_server") + [
"-log_dir",
environment.vtlogroot,
"-port",
str(rpc_port),
"-vtctl_client_protocol",
protocols_flavor().vtctl_client_protocol(),
"-vtworker_client_protocol",
protocols_flavor().vtworker_client_protocol(),
]
if auto_log:
args.append("--stderrthreshold=%s" % get_log_level())
return run_bg(args), rpc_port
for proc in proc_list:
if proc.returncode:
if options.verbose >= 1 and proc.returncode not in (-9,):
sys.stderr.write("proc failed: %s %s\n" % (proc.returncode, proc.args))
if raise_on_error:
raise CalledProcessError(proc.returncode, ' '.join(proc.args))
def run_procs(cmds, raise_on_error=True):
procs = []
for cmd in cmds:
procs.append(run_bg(cmd))
wait_procs(procs, raise_on_error=raise_on_error)
# background zk process
# (note the zkocc addresses will only work with an extra zkocc process)
zk_port_base = environment.reserve_ports(3)
def zk_setup(add_bad_host=False):
global zk_port_base
zk_ports = ":".join([str(zk_port_base), str(zk_port_base+1), str(zk_port_base+2)])
run('%s -log_dir %s -zk.cfg 1@%s:%s init' % (environment.binary_path('zkctl'), environment.vtlogroot, hostname, zk_ports))
config = environment.tmproot+'/test-zk-client-conf.json'
with open(config, 'w') as f:
ca_server = 'localhost:%u' % (zk_port_base+2)
if add_bad_host:
ca_server += ',does.not.exists:1234'
zk_cell_mapping = {'test_nj': 'localhost:%u'%(zk_port_base+2),
'test_ny': 'localhost:%u'%(zk_port_base+2),
'test_ca': ca_server,
'global': 'localhost:%u'%(zk_port_base+2),
'test_nj:_zkocc': 'localhost:%u,localhost:%u,localhost:%u'%(environment.zkocc_port_base,environment.zkocc_port_base+1,environment.zkocc_port_base+2),
'test_ny:_zkocc': 'localhost:%u'%(environment.zkocc_port_base),
def __init__(self):
self.port = environment.reserve_ports(1)
self.schema_change_dir = os.path.join(environment.tmproot,
'schema_change_test')
if protocols_flavor().vtctl_client_protocol() == 'grpc':
self.grpc_port = environment.reserve_ports(1)
def __init__(self):
self.port = environment.reserve_ports(1)
def _test_reparent_graceful(self, shard_id):
utils.run_vtctl('CreateKeyspace test_keyspace')
# create the database so vttablets start, as they are serving
tablet_62344.create_db('vt_test_keyspace')
tablet_62044.create_db('vt_test_keyspace')
tablet_41983.create_db('vt_test_keyspace')
tablet_31981.create_db('vt_test_keyspace')
# Start up a master mysql and vttablet
tablet_62344.init_tablet('master',
'test_keyspace',
shard_id,
start=True)
if environment.topo_server_implementation == 'zookeeper':
shard = utils.run_vtctl_json(
['GetShard', 'test_keyspace/' + shard_id])
self.assertEqual(
shard['Cells'], ['test_nj'],
'wrong list of cell in Shard: %s' % str(shard['Cells']))
# Create a few slaves for testing reparenting.
tablet_62044.init_tablet('replica',
'test_keyspace',
shard_id,
start=True,
wait_for_start=False)
tablet_41983.init_tablet('replica',
'test_keyspace',
shard_id,
start=True,
wait_for_start=False)
tablet_31981.init_tablet('replica',
'test_keyspace',
shard_id,
start=True,
wait_for_start=False)
for t in [tablet_62044, tablet_41983, tablet_31981]:
t.wait_for_vttablet_state("SERVING")
if environment.topo_server_implementation == 'zookeeper':
shard = utils.run_vtctl_json(
['GetShard', 'test_keyspace/' + shard_id])
self.assertEqual(
shard['Cells'], ['test_nj', 'test_ny'],
'wrong list of cell in Shard: %s' % str(shard['Cells']))
# Recompute the shard layout node - until you do that, it might not be valid.
utils.run_vtctl('RebuildShardGraph test_keyspace/' + shard_id)
utils.validate_topology()
# Force the slaves to reparent assuming that all the datasets are identical.
for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
t.reset_replication()
utils.pause("force ReparentShard?")
utils.run_vtctl('ReparentShard -force test_keyspace/%s %s' %
(shard_id, tablet_62344.tablet_alias))
utils.validate_topology(ping_tablets=True)
self._check_db_addr(shard_id, 'master', tablet_62344.port)
# Convert two replica to spare. That should leave only one node serving traffic,
# but still needs to appear in the replication graph.
utils.run_vtctl(
['ChangeSlaveType', tablet_41983.tablet_alias, 'spare'])
utils.run_vtctl(
['ChangeSlaveType', tablet_31981.tablet_alias, 'spare'])
utils.validate_topology()
self._check_db_addr(shard_id, 'replica', tablet_62044.port)
# Run this to make sure it succeeds.
utils.run_vtctl('ShardReplicationPositions test_keyspace/%s' %
shard_id,
stdout=utils.devnull)
# Perform a graceful reparent operation.
utils.pause("graceful ReparentShard?")
utils.run_vtctl('ReparentShard test_keyspace/%s %s' %
(shard_id, tablet_62044.tablet_alias),
auto_log=True)
utils.validate_topology()
self._check_db_addr(shard_id, 'master', tablet_62044.port)
tablet.kill_tablets(
[tablet_62344, tablet_62044, tablet_41983, tablet_31981])
# Test address correction.
new_port = environment.reserve_ports(1)
tablet_62044.start_vttablet(port=new_port)
# Wait a moment for address to reregister.
time.sleep(1.0)
self._check_db_addr(shard_id, 'master', new_port)
tablet_62044.kill_vttablet()
def test_standalone(self):
"""Sample test for run_local_database.py as a standalone process."""
topology = vttest_pb2.VTTestTopology()
keyspace = topology.keyspaces.add(name='test_keyspace')
keyspace.replica_count = 2
keyspace.rdonly_count = 1
keyspace.shards.add(name='-80')
keyspace.shards.add(name='80-')
topology.keyspaces.add(name='redirect', served_from='test_keyspace')
# launch a backend database based on the provided topology and schema
port = environment.reserve_ports(1)
args = [
environment.run_local_database,
'--port',
str(port),
'--proto_topo',
text_format.MessageToString(topology, as_one_line=True),
'--schema_dir',
os.path.join(environment.vttop, 'test', 'vttest_schema'),
'--web_dir',
environment.vttop + '/web/vtctld',
]
sp = subprocess.Popen(args,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
config = json.loads(sp.stdout.readline())
# gather the vars for the vtgate process
url = 'http://localhost:%d/debug/vars' % config['port']
f = urllib.urlopen(url)
data = f.read()
f.close()
json_vars = json.loads(data)
self.assertIn('vtcombo', json_vars['cmdline'][0])
# build the vtcombo address and protocol
protocol = protocols_flavor().vttest_protocol()
if protocol == 'grpc':
vtgate_addr = 'localhost:%d' % config['grpc_port']
else:
vtgate_addr = 'localhost:%d' % config['port']
conn_timeout = 30.0
utils.pause('Paused test after vtcombo was started.\n'
'For manual testing, connect to vtgate at: %s '
'using protocol: %s.\n'
'Press enter to continue.' % (vtgate_addr, protocol))
# Remember the current timestamp after we sleep for a bit, so we
# can use it for UpdateStream later.
time.sleep(2)
before_insert = long(time.time())
# Connect to vtgate.
conn = vtgate_client.connect(protocol, vtgate_addr, conn_timeout)
# Insert a row.
row_id = 123
keyspace_id = get_keyspace_id(row_id)
cursor = conn.cursor(tablet_type='master',
keyspace='test_keyspace',
keyspace_ids=[pack_kid(keyspace_id)],
writable=True)
cursor.begin()
insert = ('insert into test_table (id, msg, keyspace_id) values (:id, '
':msg, :keyspace_id)')
bind_variables = {
'id': row_id,
'msg': 'test %s' % row_id,
'keyspace_id': keyspace_id,
}
cursor.execute(insert, bind_variables)
cursor.commit()
# Read the row back.
cursor.execute('select * from test_table where id=:id', {'id': row_id})
result = cursor.fetchall()
self.assertEqual(result[0][1], 'test 123')
# try to insert again, see if we get the right integrity error exception
# (this is meant to test vtcombo properly returns exceptions, and to a
# lesser extent that the python client converts it properly)
cursor.begin()
with self.assertRaises(dbexceptions.IntegrityError):
cursor.execute(insert, bind_variables)
cursor.rollback()
# Insert a bunch of rows with long msg values.
bind_variables['msg'] = 'x' * 64
id_start = 1000
rowcount = 500
cursor.begin()
for i in xrange(id_start, id_start + rowcount):
bind_variables['id'] = i
bind_variables['keyspace_id'] = get_keyspace_id(i)
cursor.execute(insert, bind_variables)
cursor.commit()
cursor.close()
# Try to fetch a large number of rows, from a rdonly
# (more than one streaming result packet).
stream_cursor = conn.cursor(
tablet_type='rdonly',
keyspace='test_keyspace',
keyspace_ids=[pack_kid(keyspace_id)],
cursorclass=vtgate_cursor.StreamVTGateCursor)
stream_cursor.execute('select * from test_table where id >= :id_start',
{'id_start': id_start})
self.assertEqual(rowcount, len(list(stream_cursor.fetchall())))
stream_cursor.close()
# try to read a row using the redirected keyspace, to a replica this time
row_id = 123
keyspace_id = get_keyspace_id(row_id)
cursor = conn.cursor(tablet_type='replica',
keyspace='redirect',
keyspace_ids=[pack_kid(keyspace_id)])
cursor.execute('select * from test_table where id=:id', {'id': row_id})
result = cursor.fetchall()
self.assertEqual(result[0][1], 'test 123')
cursor.close()
# Try to get the update stream from the connection. This makes
# sure that part works as well.
count = 0
for (event, _) in conn.update_stream('test_keyspace',
topodata_pb2.MASTER,
timestamp=before_insert,
shard='-80'):
for statement in event.statements:
if statement.table_name == 'test_table':
count += 1
if count == rowcount + 1:
# We're getting the initial value, plus the 500 updates.
break
# Insert a sentinel value into the second shard.
row_id = 0x8100000000000000
keyspace_id = get_keyspace_id(row_id)
cursor = conn.cursor(tablet_type='master',
keyspace='test_keyspace',
keyspace_ids=[pack_kid(keyspace_id)],
writable=True)
cursor.begin()
bind_variables = {
'id': row_id,
'msg': 'test %s' % row_id,
'keyspace_id': keyspace_id,
}
cursor.execute(insert, bind_variables)
cursor.commit()
cursor.close()
# Try to connect to an update stream on the other shard.
# We may get some random update stream events, but we should not get any
# event that's related to the first shard. Only events related to
# the Insert we just did.
found = False
for (event, _) in conn.update_stream('test_keyspace',
topodata_pb2.MASTER,
timestamp=before_insert,
shard='80-'):
for statement in event.statements:
self.assertEqual(statement.table_name, 'test_table')
fields, rows = proto3_encoding.convert_stream_event_statement(
statement)
self.assertEqual(fields[0], 'id')
self.assertEqual(rows[0][0], row_id)
found = True
if found:
break
# Clean up the connection
conn.close()
# Test we can connect to vtcombo for vtctl actions
protocol = protocols_flavor().vtctl_python_client_protocol()
if protocol == 'grpc':
vtgate_addr = 'localhost:%d' % config['grpc_port']
else:
vtgate_addr = 'localhost:%d' % config['port']
out, _ = utils.run(environment.binary_args('vtctlclient') + [
'-vtctl_client_protocol',
protocol,
'-server',
vtgate_addr,
'-stderrthreshold',
'0',
'ListAllTablets',
'test',
],
trap_output=True)
num_master = 0
num_replica = 0
num_rdonly = 0
num_dash_80 = 0
num_80_dash = 0
for line in out.splitlines():
parts = line.split()
self.assertEqual(parts[1], 'test_keyspace',
'invalid keyspace in line: %s' % line)
if parts[3] == 'master':
num_master += 1
elif parts[3] == 'replica':
num_replica += 1
elif parts[3] == 'rdonly':
num_rdonly += 1
else:
self.fail('invalid tablet type in line: %s' % line)
if parts[2] == '-80':
num_dash_80 += 1
elif parts[2] == '80-':
num_80_dash += 1
else:
self.fail('invalid shard name in line: %s' % line)
self.assertEqual(num_master, 2)
self.assertEqual(num_replica, 2)
self.assertEqual(num_rdonly, 2)
self.assertEqual(num_dash_80, 3)
self.assertEqual(num_80_dash, 3)
# and we're done, clean-up process
sp.stdin.write('\n')
sp.wait()
for proc in proc_list:
if proc.returncode:
if options.verbose >= 1 and proc.returncode not in (-9,):
sys.stderr.write("proc failed: %s %s\n" % (proc.returncode, proc.args))
if raise_on_error:
raise CalledProcessError(proc.returncode, ' '.join(proc.args))
def run_procs(cmds, raise_on_error=True):
procs = []
for cmd in cmds:
procs.append(run_bg(cmd))
wait_procs(procs, raise_on_error=raise_on_error)
# background zk process
# (note the zkocc addresses will only work with an extra zkocc process)
zk_port_base = environment.reserve_ports(3)
def zk_setup(add_bad_host=False):
global zk_port_base
zk_ports = ":".join([str(zk_port_base), str(zk_port_base+1), str(zk_port_base+2)])
run('%s -log_dir %s -zk.cfg 1@%s:%s init' % (environment.binary_argstr('zkctl'), environment.vtlogroot, hostname, zk_ports))
config = environment.tmproot+'/test-zk-client-conf.json'
with open(config, 'w') as f:
ca_server = 'localhost:%u' % (zk_port_base+2)
if add_bad_host:
ca_server += ',does.not.exists:1234'
zk_cell_mapping = {'test_nj': 'localhost:%u'%(zk_port_base+2),
'test_ny': 'localhost:%u'%(zk_port_base+2),
'test_ca': ca_server,
'global': 'localhost:%u'%(zk_port_base+2),
'test_nj:_zkocc': 'localhost:%u,localhost:%u,localhost:%u'%(environment.zkocc_port_base,environment.zkocc_port_base+1,environment.zkocc_port_base+2),
'test_ny:_zkocc': 'localhost:%u'%(environment.zkocc_port_base),
def _test_reparent_graceful(self, shard_id):
# create the database so vttablets start, as they are serving
tablet_62344.create_db('vt_test_keyspace')
tablet_62044.create_db('vt_test_keyspace')
tablet_41983.create_db('vt_test_keyspace')
tablet_31981.create_db('vt_test_keyspace')
# Start up a master mysql and vttablet
tablet_62344.init_tablet('replica',
'test_keyspace',
shard_id,
start=True)
shard = utils.run_vtctl_json(['GetShard', 'test_keyspace/' + shard_id])
self.assertEqual(
shard['cells'], ['test_nj'],
'wrong list of cell in Shard: %s' % str(shard['cells']))
# Create a few slaves for testing reparenting.
tablet_62044.init_tablet('replica',
'test_keyspace',
shard_id,
start=True,
wait_for_start=False)
tablet_41983.init_tablet('replica',
'test_keyspace',
shard_id,
start=True,
wait_for_start=False)
tablet_31981.init_tablet('replica',
'test_keyspace',
shard_id,
start=True,
wait_for_start=False)
for t in [tablet_62044, tablet_41983, tablet_31981]:
t.wait_for_vttablet_state('NOT_SERVING')
shard = utils.run_vtctl_json(['GetShard', 'test_keyspace/' + shard_id])
self.assertEqual(
shard['cells'], ['test_nj', 'test_ny'],
'wrong list of cell in Shard: %s' % str(shard['cells']))
# Force the slaves to reparent assuming that all the datasets are
# identical.
utils.run_vtctl([
'InitShardMaster', '-force', 'test_keyspace/' + shard_id,
tablet_62344.tablet_alias
])
utils.validate_topology(ping_tablets=True)
tablet_62344.mquery('vt_test_keyspace', self._create_vt_insert_test)
self._check_master_tablet(tablet_62344)
utils.validate_topology()
# Run this to make sure it succeeds.
stdout, _ = utils.run_vtctl(
['ShardReplicationPositions', 'test_keyspace/' + shard_id],
trap_output=True)
lines = stdout.splitlines()
self.assertEqual(len(lines), 4) # one master, three slaves
self.assertIn('master', lines[0]) # master first
# Perform a graceful reparent operation.
utils.run_vtctl([
'PlannedReparentShard', '-keyspace_shard', 'test_keyspace/' +
shard_id, '-new_master', tablet_62044.tablet_alias
],
auto_log=True)
utils.validate_topology()
self._check_master_tablet(tablet_62044)
# insert data into the new master, check the connected slaves work
self._populate_vt_insert_test(tablet_62044, 1)
self._check_vt_insert_test(tablet_41983, 1)
self._check_vt_insert_test(tablet_62344, 1)
tablet.kill_tablets(
[tablet_62344, tablet_62044, tablet_41983, tablet_31981])
# Test address correction.
new_port = environment.reserve_ports(1)
tablet_62044.start_vttablet(port=new_port)
# Wait until the new address registers.
timeout = 30.0
while True:
try:
self._check_master_tablet(tablet_62044, port=new_port)
break
except protocols_flavor().client_error_exception_type():
timeout = utils.wait_step('waiting for new port to register',
timeout,
sleep_time=0.1)
tablet_62044.kill_vttablet()