def start_vttablet(self,
port=None,
wait_for_state='SERVING',
filecustomrules=None,
zkcustomrules=None,
schema_override=None,
repl_extra_flags=None,
table_acl_config=None,
lameduck_period=None,
security_policy=None,
full_mycnf_args=False,
extra_args=None,
extra_env=None,
include_mysql_port=True,
init_tablet_type=None,
init_keyspace=None,
init_shard=None,
init_db_name_override=None,
supports_backups=True,
grace_period='1s',
enable_semi_sync=True):
"""Starts a vttablet process, and returns it.
The process is also saved in self.proc, so it's easy to kill as well.
Returns:
the process that was started.
"""
args = environment.binary_args('vttablet')
# Use 'localhost' as hostname because Travis CI worker hostnames
# are too long for MySQL replication.
args.extend(['-tablet_hostname', 'localhost'])
args.extend(['-tablet-path', self.tablet_alias])
args.extend(environment.topo_server().flags())
args.extend([
'-binlog_player_protocol',
protocols_flavor().binlog_player_protocol()
])
args.extend([
'-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol()
])
args.extend(
['-tablet_protocol',
protocols_flavor().tabletconn_protocol()])
args.extend(['-binlog_player_healthcheck_topology_refresh', '1s'])
args.extend(['-binlog_player_healthcheck_retry_delay', '1s'])
args.extend(['-binlog_player_retry_delay', '1s'])
args.extend(
['-pid_file',
os.path.join(self.tablet_dir, 'vttablet.pid')])
# always enable_replication_reporter with somewhat short values for tests
args.extend(['-health_check_interval', '2s'])
args.extend(['-enable_replication_reporter'])
args.extend(['-degraded_threshold', '5s'])
args.extend(['-watch_replication_stream'])
if enable_semi_sync:
args.append('-enable_semi_sync')
if self.use_mysqlctld:
args.extend([
'-mysqlctl_socket',
os.path.join(self.tablet_dir, 'mysqlctl.sock')
])
if full_mycnf_args:
# this flag is used to specify all the mycnf_ flags, to make
# sure that code works.
relay_log_path = os.path.join(
self.tablet_dir, 'relay-logs',
'vt-%010d-relay-bin' % self.tablet_uid)
args.extend([
'-mycnf_server_id',
str(self.tablet_uid),
'-mycnf_data_dir',
os.path.join(self.tablet_dir, 'data'),
'-mycnf_innodb_data_home_dir',
os.path.join(self.tablet_dir, 'innodb', 'data'),
'-mycnf_innodb_log_group_home_dir',
os.path.join(self.tablet_dir, 'innodb', 'logs'),
'-mycnf_socket_file',
os.path.join(self.tablet_dir, 'mysql.sock'),
'-mycnf_error_log_path',
os.path.join(self.tablet_dir, 'error.log'),
'-mycnf_slow_log_path',
os.path.join(self.tablet_dir, 'slow-query.log'),
'-mycnf_relay_log_path',
relay_log_path,
'-mycnf_relay_log_index_path',
relay_log_path + '.index',
'-mycnf_relay_log_info_path',
os.path.join(self.tablet_dir, 'relay-logs', 'relay-log.info'),
'-mycnf_bin_log_path',
os.path.join(self.tablet_dir, 'bin-logs',
'vt-%010d-bin' % self.tablet_uid),
'-mycnf_master_info_file',
os.path.join(self.tablet_dir, 'master.info'),
'-mycnf_pid_file',
os.path.join(self.tablet_dir, 'mysql.pid'),
'-mycnf_tmp_dir',
os.path.join(self.tablet_dir, 'tmp'),
'-mycnf_slave_load_tmp_dir',
os.path.join(self.tablet_dir, 'tmp'),
])
if include_mysql_port:
args.extend(['-mycnf_mysql_port', str(self.mysql_port)])
# this is used to run InitTablet as part of the vttablet startup
if init_tablet_type:
self.tablet_type = init_tablet_type
args.extend(['-init_tablet_type', init_tablet_type])
if init_keyspace:
self.keyspace = init_keyspace
self.shard = init_shard
args.extend(
['-init_keyspace', init_keyspace, '-init_shard', init_shard])
if init_db_name_override:
self.dbname = init_db_name_override
args.extend(['-init_db_name_override', init_db_name_override])
else:
self.dbname = 'vt_' + init_keyspace
if supports_backups:
args.extend(['-restore_from_backup'] + get_backup_storage_flags())
if extra_args:
args.extend(extra_args)
args.extend([
'-port',
'%s' % (port or self.port), '-log_dir', environment.vtlogroot
])
self._add_dbconfigs(args, repl_extra_flags)
if filecustomrules:
args.extend(['-filecustomrules', filecustomrules])
if zkcustomrules:
args.extend(['-zkcustomrules', zkcustomrules])
if schema_override:
args.extend(['-schema-override', schema_override])
if table_acl_config:
args.extend(['-table-acl-config', table_acl_config])
args.extend(['-queryserver-config-strict-table-acl'])
if protocols_flavor().service_map():
args.extend(
['-service_map', ','.join(protocols_flavor().service_map())])
if self.grpc_enabled():
args.extend(['-grpc_port', str(self.grpc_port)])
if lameduck_period:
args.extend(environment.lameduck_flag(lameduck_period))
if grace_period:
args.extend(['-serving_state_grace_period', grace_period])
if security_policy:
args.extend(['-security_policy', security_policy])
if extra_args:
args.extend(extra_args)
args.extend(['-enable-autocommit'])
stderr_fd = open(
os.path.join(environment.vtlogroot,
'vttablet-%d.stderr' % self.tablet_uid), 'w')
# increment count only the first time
if not self.proc:
Tablet.tablets_running += 1
self.proc = utils.run_bg(args, stderr=stderr_fd, extra_env=extra_env)
log_message = ('Started vttablet: %s (%s) with pid: %s - Log files: '
'%s/vttablet.*.{INFO,WARNING,ERROR,FATAL}.*.%s' %
(self.tablet_uid, self.tablet_alias, self.proc.pid,
environment.vtlogroot, self.proc.pid))
# This may race with the stderr output from the process (though
# that's usually empty).
stderr_fd.write(log_message + '\n')
stderr_fd.close()
logging.debug(log_message)
# wait for query service to be in the right state
if wait_for_state:
self.wait_for_vttablet_state(wait_for_state, port=port)
if self.tablet_index is not None:
topo_server().update_addr('test_' + self.cell, self.keyspace,
self.shard, self.tablet_index,
(port or self.port))
return self.proc
def start_vttablet(
self, port=None, memcache=False,
wait_for_state='SERVING', filecustomrules=None, zkcustomrules=None,
schema_override=None,
repl_extra_flags=None, table_acl_config=None,
lameduck_period=None, security_policy=None,
full_mycnf_args=False,
extra_args=None, extra_env=None, include_mysql_port=True,
init_tablet_type=None, init_keyspace=None,
init_shard=None, init_db_name_override=None,
supports_backups=False, grace_period='1s', enable_semi_sync=True):
"""Starts a vttablet process, and returns it.
The process is also saved in self.proc, so it's easy to kill as well.
Returns:
the process that was started.
"""
args = environment.binary_args('vttablet')
# Use 'localhost' as hostname because Travis CI worker hostnames
# are too long for MySQL replication.
args.extend(['-tablet_hostname', 'localhost'])
args.extend(['-tablet-path', self.tablet_alias])
args.extend(environment.topo_server().flags())
args.extend(['-binlog_player_protocol',
protocols_flavor().binlog_player_protocol()])
args.extend(['-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol()])
args.extend(['-tablet_protocol', protocols_flavor().tabletconn_protocol()])
args.extend(['-binlog_player_healthcheck_topology_refresh', '1s'])
args.extend(['-binlog_player_healthcheck_retry_delay', '1s'])
args.extend(['-binlog_player_retry_delay', '1s'])
args.extend(['-pid_file', os.path.join(self.tablet_dir, 'vttablet.pid')])
# always enable_replication_lag_check with somewhat short values for tests
args.extend(['-health_check_interval', '2s'])
args.extend(['-enable_replication_lag_check'])
args.extend(['-degraded_threshold', '5s'])
if enable_semi_sync:
args.append('-enable_semi_sync')
if self.use_mysqlctld:
args.extend(
['-mysqlctl_socket', os.path.join(self.tablet_dir, 'mysqlctl.sock')])
if full_mycnf_args:
# this flag is used to specify all the mycnf_ flags, to make
# sure that code works.
relay_log_path = os.path.join(self.tablet_dir, 'relay-logs',
'vt-%010d-relay-bin' % self.tablet_uid)
args.extend([
'-mycnf_server_id', str(self.tablet_uid),
'-mycnf_data_dir', os.path.join(self.tablet_dir, 'data'),
'-mycnf_innodb_data_home_dir', os.path.join(self.tablet_dir,
'innodb', 'data'),
'-mycnf_innodb_log_group_home_dir', os.path.join(self.tablet_dir,
'innodb', 'logs'),
'-mycnf_socket_file', os.path.join(self.tablet_dir, 'mysql.sock'),
'-mycnf_error_log_path', os.path.join(self.tablet_dir, 'error.log'),
'-mycnf_slow_log_path', os.path.join(self.tablet_dir,
'slow-query.log'),
'-mycnf_relay_log_path', relay_log_path,
'-mycnf_relay_log_index_path', relay_log_path + '.index',
'-mycnf_relay_log_info_path', os.path.join(self.tablet_dir,
'relay-logs',
'relay-log.info'),
'-mycnf_bin_log_path', os.path.join(
self.tablet_dir, 'bin-logs', 'vt-%010d-bin' % self.tablet_uid),
'-mycnf_master_info_file', os.path.join(self.tablet_dir,
'master.info'),
'-mycnf_pid_file', os.path.join(self.tablet_dir, 'mysql.pid'),
'-mycnf_tmp_dir', os.path.join(self.tablet_dir, 'tmp'),
'-mycnf_slave_load_tmp_dir', os.path.join(self.tablet_dir, 'tmp'),
])
if include_mysql_port:
args.extend(['-mycnf_mysql_port', str(self.mysql_port)])
# this is used to run InitTablet as part of the vttablet startup
if init_tablet_type:
self.tablet_type = init_tablet_type
args.extend(['-init_tablet_type', init_tablet_type])
if init_keyspace:
self.keyspace = init_keyspace
self.shard = init_shard
args.extend(['-init_keyspace', init_keyspace,
'-init_shard', init_shard])
if init_db_name_override:
self.dbname = init_db_name_override
args.extend(['-init_db_name_override', init_db_name_override])
else:
self.dbname = 'vt_' + init_keyspace
if supports_backups:
args.extend(['-restore_from_backup'] + get_backup_storage_flags())
if extra_args:
args.extend(extra_args)
args.extend(['-port', '%s' % (port or self.port),
'-log_dir', environment.vtlogroot])
self._add_dbconfigs(args, repl_extra_flags)
if memcache:
args.extend(['-rowcache-bin', environment.memcached_bin()])
memcache_socket = os.path.join(self.tablet_dir, 'memcache.sock')
args.extend(['-rowcache-socket', memcache_socket])
args.extend(['-enable-rowcache'])
if filecustomrules:
args.extend(['-filecustomrules', filecustomrules])
if zkcustomrules:
args.extend(['-zkcustomrules', zkcustomrules])
if schema_override:
args.extend(['-schema-override', schema_override])
if table_acl_config:
args.extend(['-table-acl-config', table_acl_config])
args.extend(['-queryserver-config-strict-table-acl'])
if protocols_flavor().service_map():
args.extend(['-service_map', ','.join(protocols_flavor().service_map())])
if self.grpc_enabled():
args.extend(['-grpc_port', str(self.grpc_port)])
if lameduck_period:
args.extend(['-lameduck-period', lameduck_period])
if grace_period:
args.extend(['-serving_state_grace_period', grace_period])
if security_policy:
args.extend(['-security_policy', security_policy])
if extra_args:
args.extend(extra_args)
args.extend(['-enable-autocommit'])
stderr_fd = open(
os.path.join(environment.vtlogroot, 'vttablet-%d.stderr' %
self.tablet_uid), 'w')
# increment count only the first time
if not self.proc:
Tablet.tablets_running += 1
self.proc = utils.run_bg(args, stderr=stderr_fd, extra_env=extra_env)
log_message = (
'Started vttablet: %s (%s) with pid: %s - Log files: '
'%s/vttablet.*.{INFO,WARNING,ERROR,FATAL}.*.%s' %
(self.tablet_uid, self.tablet_alias, self.proc.pid,
environment.vtlogroot, self.proc.pid))
# This may race with the stderr output from the process (though
# that's usually empty).
stderr_fd.write(log_message + '\n')
stderr_fd.close()
logging.debug(log_message)
# wait for query service to be in the right state
if wait_for_state:
self.wait_for_vttablet_state(wait_for_state, port=port)
if self.tablet_index is not None:
topo_server().update_addr(
'test_'+self.cell, self.keyspace, self.shard,
self.tablet_index, (port or self.port))
return self.proc
def start_vttablet(
self,
port=None,
wait_for_state="SERVING",
filecustomrules=None,
zkcustomrules=None,
schema_override=None,
repl_extra_flags=None,
table_acl_config=None,
lameduck_period=None,
security_policy=None,
full_mycnf_args=False,
extra_args=None,
extra_env=None,
include_mysql_port=True,
init_tablet_type=None,
init_keyspace=None,
init_shard=None,
init_db_name_override=None,
supports_backups=True,
grace_period="1s",
enable_semi_sync=True,
):
"""Starts a vttablet process, and returns it.
The process is also saved in self.proc, so it's easy to kill as well.
Returns:
the process that was started.
"""
args = environment.binary_args("vttablet")
# Use 'localhost' as hostname because Travis CI worker hostnames
# are too long for MySQL replication.
args.extend(["-tablet_hostname", "localhost"])
args.extend(["-tablet-path", self.tablet_alias])
args.extend(environment.topo_server().flags())
args.extend(["-binlog_player_protocol", protocols_flavor().binlog_player_protocol()])
args.extend(["-tablet_manager_protocol", protocols_flavor().tablet_manager_protocol()])
args.extend(["-tablet_protocol", protocols_flavor().tabletconn_protocol()])
args.extend(["-binlog_player_healthcheck_topology_refresh", "1s"])
args.extend(["-binlog_player_healthcheck_retry_delay", "1s"])
args.extend(["-binlog_player_retry_delay", "1s"])
args.extend(["-pid_file", os.path.join(self.tablet_dir, "vttablet.pid")])
# always enable_replication_reporter with somewhat short values for tests
args.extend(["-health_check_interval", "2s"])
args.extend(["-enable_replication_reporter"])
args.extend(["-degraded_threshold", "5s"])
args.extend(["-watch_replication_stream"])
if enable_semi_sync:
args.append("-enable_semi_sync")
if self.use_mysqlctld:
args.extend(["-mysqlctl_socket", os.path.join(self.tablet_dir, "mysqlctl.sock")])
if full_mycnf_args:
# this flag is used to specify all the mycnf_ flags, to make
# sure that code works.
relay_log_path = os.path.join(self.tablet_dir, "relay-logs", "vt-%010d-relay-bin" % self.tablet_uid)
args.extend(
[
"-mycnf_server_id",
str(self.tablet_uid),
"-mycnf_data_dir",
os.path.join(self.tablet_dir, "data"),
"-mycnf_innodb_data_home_dir",
os.path.join(self.tablet_dir, "innodb", "data"),
"-mycnf_innodb_log_group_home_dir",
os.path.join(self.tablet_dir, "innodb", "logs"),
"-mycnf_socket_file",
os.path.join(self.tablet_dir, "mysql.sock"),
"-mycnf_error_log_path",
os.path.join(self.tablet_dir, "error.log"),
"-mycnf_slow_log_path",
os.path.join(self.tablet_dir, "slow-query.log"),
"-mycnf_relay_log_path",
relay_log_path,
"-mycnf_relay_log_index_path",
relay_log_path + ".index",
"-mycnf_relay_log_info_path",
os.path.join(self.tablet_dir, "relay-logs", "relay-log.info"),
"-mycnf_bin_log_path",
os.path.join(self.tablet_dir, "bin-logs", "vt-%010d-bin" % self.tablet_uid),
"-mycnf_master_info_file",
os.path.join(self.tablet_dir, "master.info"),
"-mycnf_pid_file",
os.path.join(self.tablet_dir, "mysql.pid"),
"-mycnf_tmp_dir",
os.path.join(self.tablet_dir, "tmp"),
"-mycnf_slave_load_tmp_dir",
os.path.join(self.tablet_dir, "tmp"),
]
)
if include_mysql_port:
args.extend(["-mycnf_mysql_port", str(self.mysql_port)])
# this is used to run InitTablet as part of the vttablet startup
if init_tablet_type:
self.tablet_type = init_tablet_type
args.extend(["-init_tablet_type", init_tablet_type])
if init_keyspace:
self.keyspace = init_keyspace
self.shard = init_shard
args.extend(["-init_keyspace", init_keyspace, "-init_shard", init_shard])
if init_db_name_override:
self.dbname = init_db_name_override
args.extend(["-init_db_name_override", init_db_name_override])
else:
self.dbname = "vt_" + init_keyspace
if supports_backups:
args.extend(["-restore_from_backup"] + get_backup_storage_flags())
if extra_args:
args.extend(extra_args)
args.extend(["-port", "%s" % (port or self.port), "-log_dir", environment.vtlogroot])
self._add_dbconfigs(args, repl_extra_flags)
if filecustomrules:
args.extend(["-filecustomrules", filecustomrules])
if zkcustomrules:
args.extend(["-zkcustomrules", zkcustomrules])
if schema_override:
args.extend(["-schema-override", schema_override])
if table_acl_config:
args.extend(["-table-acl-config", table_acl_config])
args.extend(["-queryserver-config-strict-table-acl"])
if protocols_flavor().service_map():
args.extend(["-service_map", ",".join(protocols_flavor().service_map())])
if self.grpc_enabled():
args.extend(["-grpc_port", str(self.grpc_port)])
if lameduck_period:
args.extend(environment.lameduck_flag(lameduck_period))
if grace_period:
args.extend(["-serving_state_grace_period", grace_period])
if security_policy:
args.extend(["-security_policy", security_policy])
if extra_args:
args.extend(extra_args)
args.extend(["-enable-autocommit"])
stderr_fd = open(os.path.join(environment.vtlogroot, "vttablet-%d.stderr" % self.tablet_uid), "w")
# increment count only the first time
if not self.proc:
Tablet.tablets_running += 1
self.proc = utils.run_bg(args, stderr=stderr_fd, extra_env=extra_env)
log_message = (
"Started vttablet: %s (%s) with pid: %s - Log files: "
"%s/vttablet.*.{INFO,WARNING,ERROR,FATAL}.*.%s"
% (self.tablet_uid, self.tablet_alias, self.proc.pid, environment.vtlogroot, self.proc.pid)
)
# This may race with the stderr output from the process (though
# that's usually empty).
stderr_fd.write(log_message + "\n")
stderr_fd.close()
logging.debug(log_message)
# wait for query service to be in the right state
if wait_for_state:
self.wait_for_vttablet_state(wait_for_state, port=port)
if self.tablet_index is not None:
topo_server().update_addr(
"test_" + self.cell, self.keyspace, self.shard, self.tablet_index, (port or self.port)
)
return self.proc
def start_vttablet(
self,
port=None,
wait_for_state='SERVING',
topocustomrule_path=None,
schema_override=None,
repl_extra_flags=None,
table_acl_config=None,
lameduck_period=None,
security_policy=None,
full_mycnf_args=False,
extra_args=None,
extra_env=None,
include_mysql_port=True,
init_tablet_type=None,
init_keyspace=None,
init_shard=None,
# TODO(mberlin): Assign the index automatically and remove this parameter.
tablet_index=None,
init_db_name_override=None,
binlog_use_v3_resharding_mode=True,
supports_backups=True,
grace_period='1s',
enable_semi_sync=True):
# pylint: disable=g-doc-args
"""Starts a vttablet process, and returns it.
The process is also saved in self.proc, so it's easy to kill as well.
Returns:
the process that was started.
"""
# pylint: enable=g-doc-args
args = environment.binary_args('vttablet')
# Use 'localhost' as hostname because Travis CI worker hostnames
# are too long for MySQL replication.
args.extend(['-tablet_hostname', 'localhost'])
args.extend(['-tablet-path', self.tablet_alias])
args.extend(environment.topo_server().flags())
args.extend([
'-binlog_player_protocol',
protocols_flavor().binlog_player_protocol()
])
args.extend([
'-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol()
])
args.extend(
['-tablet_protocol',
protocols_flavor().tabletconn_protocol()])
args.extend(['-vreplication_healthcheck_topology_refresh', '1s'])
args.extend(['-vreplication_healthcheck_retry_delay', '1s'])
args.extend(['-vreplication_retry_delay', '1s'])
args.extend(
['-pid_file',
os.path.join(self.tablet_dir, 'vttablet.pid')])
# always enable_replication_reporter with somewhat short values for tests
args.extend(['-health_check_interval', '2s'])
args.extend(['-enable_replication_reporter'])
args.extend(['-degraded_threshold', '5s'])
args.extend(['-lock_tables_timeout', '5s'])
args.extend(['-watch_replication_stream'])
if enable_semi_sync:
args.append('-enable_semi_sync')
# Remember the setting in case a test wants to know it.
self.enable_semi_sync = enable_semi_sync
if self.use_mysqlctld:
args.extend([
'-mysqlctl_socket',
os.path.join(self.tablet_dir, 'mysqlctl.sock')
])
if self.external_mysql:
args.extend(['-db_host', '127.0.0.1'])
args.extend(['-db_port', str(self.mysql_port)])
args.append('-disable_active_reparents')
if full_mycnf_args:
# this flag is used to specify all the mycnf_ flags, to make
# sure that code works.
relay_log_path = os.path.join(
self.tablet_dir, 'relay-logs',
'vt-%010d-relay-bin' % self.tablet_uid)
args.extend([
'-mycnf_server_id',
str(self.tablet_uid),
'-mycnf_data_dir',
os.path.join(self.tablet_dir, 'data'),
'-mycnf_innodb_data_home_dir',
os.path.join(self.tablet_dir, 'innodb', 'data'),
'-mycnf_innodb_log_group_home_dir',
os.path.join(self.tablet_dir, 'innodb', 'logs'),
'-mycnf_socket_file',
os.path.join(self.tablet_dir, 'mysql.sock'),
'-mycnf_error_log_path',
os.path.join(self.tablet_dir, 'error.log'),
'-mycnf_slow_log_path',
os.path.join(self.tablet_dir, 'slow-query.log'),
'-mycnf_relay_log_path',
relay_log_path,
'-mycnf_relay_log_index_path',
relay_log_path + '.index',
'-mycnf_relay_log_info_path',
os.path.join(self.tablet_dir, 'relay-logs', 'relay-log.info'),
'-mycnf_bin_log_path',
os.path.join(self.tablet_dir, 'bin-logs',
'vt-%010d-bin' % self.tablet_uid),
'-mycnf_master_info_file',
os.path.join(self.tablet_dir, 'master.info'),
'-mycnf_pid_file',
os.path.join(self.tablet_dir, 'mysql.pid'),
'-mycnf_tmp_dir',
os.path.join(self.tablet_dir, 'tmp'),
'-mycnf_slave_load_tmp_dir',
os.path.join(self.tablet_dir, 'tmp'),
])
if include_mysql_port:
args.extend(['-mycnf_mysql_port', str(self.mysql_port)])
# this is used to run InitTablet as part of the vttablet startup
if init_tablet_type:
self.tablet_type = init_tablet_type
args.extend(['-init_tablet_type', init_tablet_type])
if init_keyspace:
self.keyspace = init_keyspace
self.shard = init_shard
# tablet_index is required for the update_addr call below.
if self.tablet_index is None:
self.tablet_index = tablet_index
args.extend(
['-init_keyspace', init_keyspace, '-init_shard', init_shard])
if init_db_name_override:
self.dbname = init_db_name_override
args.extend(['-init_db_name_override', init_db_name_override])
else:
self.dbname = 'vt_' + init_keyspace
# Default value for this flag is True. So, add it only if it's false.
if not binlog_use_v3_resharding_mode:
args.extend(['-binlog_use_v3_resharding_mode=false'])
if supports_backups:
args.extend(['-restore_from_backup'] + get_backup_storage_flags())
# When vttablet restores from backup, it will re-generate the .cnf file.
# So we need to have EXTRA_MY_CNF set properly.
# When using mysqlctld, only mysqlctld should need EXTRA_MY_CNF.
# If any test fails without giving EXTRA_MY_CNF to vttablet,
# it means we missed some call that should run remotely on mysqlctld.
if not self.use_mysqlctld:
all_extra_my_cnf = get_all_extra_my_cnf(None)
if all_extra_my_cnf:
if not extra_env:
extra_env = {}
extra_env['EXTRA_MY_CNF'] = ':'.join(all_extra_my_cnf)
if extra_args:
args.extend(extra_args)
args.extend([
'-port',
'%s' % (port or self.port), '-log_dir', environment.vtlogroot
])
if topocustomrule_path:
args.extend(['-topocustomrule_path', topocustomrule_path])
if schema_override:
args.extend(['-schema-override', schema_override])
if table_acl_config:
args.extend(['-table-acl-config', table_acl_config])
args.extend(['-queryserver-config-strict-table-acl'])
if protocols_flavor().service_map():
args.extend(
['-service_map', ','.join(protocols_flavor().service_map())])
if self.grpc_enabled():
args.extend(['-grpc_port', str(self.grpc_port)])
args.extend([
'-grpc_max_message_size',
str(environment.grpc_max_message_size)
])
if lameduck_period:
args.extend(environment.lameduck_flag(lameduck_period))
if grace_period:
args.extend(['-serving_state_grace_period', grace_period])
if security_policy:
args.extend(['-security_policy', security_policy])
args.extend(['-enable-autocommit'])
stderr_fd = open(
os.path.join(environment.vtlogroot,
'vttablet-%d.stderr' % self.tablet_uid), 'w')
# increment count only the first time
if not self.proc:
Tablet.tablets_running += 1
self.proc = utils.run_bg(args, stderr=stderr_fd, extra_env=extra_env)
log_message = ('Started vttablet: %s (%s) with pid: %s - Log files: '
'%s/vttablet.*.{INFO,WARNING,ERROR,FATAL}.*.%s' %
(self.tablet_uid, self.tablet_alias, self.proc.pid,
environment.vtlogroot, self.proc.pid))
# This may race with the stderr output from the process (though
# that's usually empty).
stderr_fd.write(log_message + '\n')
stderr_fd.close()
logging.debug(log_message)
# wait for query service to be in the right state
if wait_for_state:
self.wait_for_vttablet_state(wait_for_state, port=port)
if self.tablet_index is not None:
topo_server().update_addr('test_' + self.cell, self.keyspace,
self.shard, self.tablet_index,
(port or self.port))
return self.proc
def start_vttablet(
self, port=None,
wait_for_state='SERVING', topocustomrule_path=None,
schema_override=None,
repl_extra_flags=None, table_acl_config=None,
lameduck_period=None, security_policy=None,
full_mycnf_args=False,
extra_args=None, extra_env=None, include_mysql_port=True,
init_tablet_type=None, init_keyspace=None, init_shard=None,
# TODO(mberlin): Assign the index automatically and remove this parameter.
tablet_index=None,
init_db_name_override=None,
binlog_use_v3_resharding_mode=True,
supports_backups=True, grace_period='1s', enable_semi_sync=True):
# pylint: disable=g-doc-args
"""Starts a vttablet process, and returns it.
The process is also saved in self.proc, so it's easy to kill as well.
Returns:
the process that was started.
"""
# pylint: enable=g-doc-args
args = environment.binary_args('vttablet')
# Use 'localhost' as hostname because Travis CI worker hostnames
# are too long for MySQL replication.
args.extend(['-tablet_hostname', 'localhost'])
args.extend(['-tablet-path', self.tablet_alias])
args.extend(environment.topo_server().flags())
args.extend(['-binlog_player_protocol',
protocols_flavor().binlog_player_protocol()])
args.extend(['-tablet_manager_protocol',
protocols_flavor().tablet_manager_protocol()])
args.extend(['-tablet_protocol', protocols_flavor().tabletconn_protocol()])
args.extend(['-binlog_player_healthcheck_topology_refresh', '1s'])
args.extend(['-binlog_player_healthcheck_retry_delay', '1s'])
args.extend(['-binlog_player_retry_delay', '1s'])
args.extend(['-pid_file', os.path.join(self.tablet_dir, 'vttablet.pid')])
# always enable_replication_reporter with somewhat short values for tests
args.extend(['-health_check_interval', '2s'])
args.extend(['-enable_replication_reporter'])
args.extend(['-degraded_threshold', '5s'])
args.extend(['-watch_replication_stream'])
if enable_semi_sync:
args.append('-enable_semi_sync')
# Remember the setting in case a test wants to know it.
self.enable_semi_sync = enable_semi_sync
if self.use_mysqlctld:
args.extend(
['-mysqlctl_socket', os.path.join(self.tablet_dir, 'mysqlctl.sock')])
if full_mycnf_args:
# this flag is used to specify all the mycnf_ flags, to make
# sure that code works.
relay_log_path = os.path.join(self.tablet_dir, 'relay-logs',
'vt-%010d-relay-bin' % self.tablet_uid)
args.extend([
'-mycnf_server_id', str(self.tablet_uid),
'-mycnf_data_dir', os.path.join(self.tablet_dir, 'data'),
'-mycnf_innodb_data_home_dir', os.path.join(self.tablet_dir,
'innodb', 'data'),
'-mycnf_innodb_log_group_home_dir', os.path.join(self.tablet_dir,
'innodb', 'logs'),
'-mycnf_socket_file', os.path.join(self.tablet_dir, 'mysql.sock'),
'-mycnf_error_log_path', os.path.join(self.tablet_dir, 'error.log'),
'-mycnf_slow_log_path', os.path.join(self.tablet_dir,
'slow-query.log'),
'-mycnf_relay_log_path', relay_log_path,
'-mycnf_relay_log_index_path', relay_log_path + '.index',
'-mycnf_relay_log_info_path', os.path.join(self.tablet_dir,
'relay-logs',
'relay-log.info'),
'-mycnf_bin_log_path', os.path.join(
self.tablet_dir, 'bin-logs', 'vt-%010d-bin' % self.tablet_uid),
'-mycnf_master_info_file', os.path.join(self.tablet_dir,
'master.info'),
'-mycnf_pid_file', os.path.join(self.tablet_dir, 'mysql.pid'),
'-mycnf_tmp_dir', os.path.join(self.tablet_dir, 'tmp'),
'-mycnf_slave_load_tmp_dir', os.path.join(self.tablet_dir, 'tmp'),
])
if include_mysql_port:
args.extend(['-mycnf_mysql_port', str(self.mysql_port)])
# this is used to run InitTablet as part of the vttablet startup
if init_tablet_type:
self.tablet_type = init_tablet_type
args.extend(['-init_tablet_type', init_tablet_type])
if init_keyspace:
self.keyspace = init_keyspace
self.shard = init_shard
# tablet_index is required for the update_addr call below.
if self.tablet_index is None:
self.tablet_index = tablet_index
args.extend(['-init_keyspace', init_keyspace,
'-init_shard', init_shard])
if init_db_name_override:
self.dbname = init_db_name_override
args.extend(['-init_db_name_override', init_db_name_override])
else:
self.dbname = 'vt_' + init_keyspace
# Default value for this flag is True. So, add it only if it's false.
if not binlog_use_v3_resharding_mode:
args.extend(['-binlog_use_v3_resharding_mode=false'])
if supports_backups:
args.extend(['-restore_from_backup'] + get_backup_storage_flags())
# When vttablet restores from backup, it will re-generate the .cnf file.
# So we need to have EXTRA_MY_CNF set properly.
all_extra_my_cnf = get_all_extra_my_cnf(None)
if all_extra_my_cnf:
if not extra_env:
extra_env = {}
extra_env['EXTRA_MY_CNF'] = ':'.join(all_extra_my_cnf)
if extra_args:
args.extend(extra_args)
args.extend(['-port', '%s' % (port or self.port),
'-log_dir', environment.vtlogroot])
self._add_dbconfigs(self.default_db_config, args, repl_extra_flags)
if topocustomrule_path:
args.extend(['-topocustomrule_path', topocustomrule_path])
if schema_override:
args.extend(['-schema-override', schema_override])
if table_acl_config:
args.extend(['-table-acl-config', table_acl_config])
args.extend(['-queryserver-config-strict-table-acl'])
if protocols_flavor().service_map():
args.extend(['-service_map', ','.join(protocols_flavor().service_map())])
if self.grpc_enabled():
args.extend(['-grpc_port', str(self.grpc_port)])
args.extend(['-grpc_max_message_size',
str(environment.grpc_max_message_size)])
if lameduck_period:
args.extend(environment.lameduck_flag(lameduck_period))
if grace_period:
args.extend(['-serving_state_grace_period', grace_period])
if security_policy:
args.extend(['-security_policy', security_policy])
args.extend(['-enable-autocommit'])
stderr_fd = open(
os.path.join(environment.vtlogroot, 'vttablet-%d.stderr' %
self.tablet_uid), 'w')
# increment count only the first time
if not self.proc:
Tablet.tablets_running += 1
self.proc = utils.run_bg(args, stderr=stderr_fd, extra_env=extra_env)
log_message = (
'Started vttablet: %s (%s) with pid: %s - Log files: '
'%s/vttablet.*.{INFO,WARNING,ERROR,FATAL}.*.%s' %
(self.tablet_uid, self.tablet_alias, self.proc.pid,
environment.vtlogroot, self.proc.pid))
# This may race with the stderr output from the process (though
# that's usually empty).
stderr_fd.write(log_message + '\n')
stderr_fd.close()
logging.debug(log_message)
# wait for query service to be in the right state
if wait_for_state:
self.wait_for_vttablet_state(wait_for_state, port=port)
if self.tablet_index is not None:
topo_server().update_addr(
'test_'+self.cell, self.keyspace, self.shard,
self.tablet_index, (port or self.port))
return self.proc
