def start_vttablet(self, port=None, wait_for_state='SERVING', filecustomrules=None, zkcustomrules=None, schema_override=None, repl_extra_flags=None, table_acl_config=None, lameduck_period=None, security_policy=None, full_mycnf_args=False, extra_args=None, extra_env=None, include_mysql_port=True, init_tablet_type=None, init_keyspace=None, init_shard=None, init_db_name_override=None, supports_backups=True, grace_period='1s', enable_semi_sync=True): """Starts a vttablet process, and returns it. The process is also saved in self.proc, so it's easy to kill as well. Returns: the process that was started. """ args = environment.binary_args('vttablet') # Use 'localhost' as hostname because Travis CI worker hostnames # are too long for MySQL replication. args.extend(['-tablet_hostname', 'localhost']) args.extend(['-tablet-path', self.tablet_alias]) args.extend(environment.topo_server().flags()) args.extend([ '-binlog_player_protocol', protocols_flavor().binlog_player_protocol() ]) args.extend([ '-tablet_manager_protocol', protocols_flavor().tablet_manager_protocol() ]) args.extend( ['-tablet_protocol', protocols_flavor().tabletconn_protocol()]) args.extend(['-binlog_player_healthcheck_topology_refresh', '1s']) args.extend(['-binlog_player_healthcheck_retry_delay', '1s']) args.extend(['-binlog_player_retry_delay', '1s']) args.extend( ['-pid_file', os.path.join(self.tablet_dir, 'vttablet.pid')]) # always enable_replication_reporter with somewhat short values for tests args.extend(['-health_check_interval', '2s']) args.extend(['-enable_replication_reporter']) args.extend(['-degraded_threshold', '5s']) args.extend(['-watch_replication_stream']) if enable_semi_sync: args.append('-enable_semi_sync') if self.use_mysqlctld: args.extend([ '-mysqlctl_socket', os.path.join(self.tablet_dir, 'mysqlctl.sock') ]) if full_mycnf_args: # this flag is used to specify all the mycnf_ flags, to make # sure that code works. relay_log_path = os.path.join( self.tablet_dir, 'relay-logs', 'vt-%010d-relay-bin' % self.tablet_uid) args.extend([ '-mycnf_server_id', str(self.tablet_uid), '-mycnf_data_dir', os.path.join(self.tablet_dir, 'data'), '-mycnf_innodb_data_home_dir', os.path.join(self.tablet_dir, 'innodb', 'data'), '-mycnf_innodb_log_group_home_dir', os.path.join(self.tablet_dir, 'innodb', 'logs'), '-mycnf_socket_file', os.path.join(self.tablet_dir, 'mysql.sock'), '-mycnf_error_log_path', os.path.join(self.tablet_dir, 'error.log'), '-mycnf_slow_log_path', os.path.join(self.tablet_dir, 'slow-query.log'), '-mycnf_relay_log_path', relay_log_path, '-mycnf_relay_log_index_path', relay_log_path + '.index', '-mycnf_relay_log_info_path', os.path.join(self.tablet_dir, 'relay-logs', 'relay-log.info'), '-mycnf_bin_log_path', os.path.join(self.tablet_dir, 'bin-logs', 'vt-%010d-bin' % self.tablet_uid), '-mycnf_master_info_file', os.path.join(self.tablet_dir, 'master.info'), '-mycnf_pid_file', os.path.join(self.tablet_dir, 'mysql.pid'), '-mycnf_tmp_dir', os.path.join(self.tablet_dir, 'tmp'), '-mycnf_slave_load_tmp_dir', os.path.join(self.tablet_dir, 'tmp'), ]) if include_mysql_port: args.extend(['-mycnf_mysql_port', str(self.mysql_port)]) # this is used to run InitTablet as part of the vttablet startup if init_tablet_type: self.tablet_type = init_tablet_type args.extend(['-init_tablet_type', init_tablet_type]) if init_keyspace: self.keyspace = init_keyspace self.shard = init_shard args.extend( ['-init_keyspace', init_keyspace, '-init_shard', init_shard]) if init_db_name_override: self.dbname = init_db_name_override args.extend(['-init_db_name_override', init_db_name_override]) else: self.dbname = 'vt_' + init_keyspace if supports_backups: args.extend(['-restore_from_backup'] + get_backup_storage_flags()) if extra_args: args.extend(extra_args) args.extend([ '-port', '%s' % (port or self.port), '-log_dir', environment.vtlogroot ]) self._add_dbconfigs(args, repl_extra_flags) if filecustomrules: args.extend(['-filecustomrules', filecustomrules]) if zkcustomrules: args.extend(['-zkcustomrules', zkcustomrules]) if schema_override: args.extend(['-schema-override', schema_override]) if table_acl_config: args.extend(['-table-acl-config', table_acl_config]) args.extend(['-queryserver-config-strict-table-acl']) if protocols_flavor().service_map(): args.extend( ['-service_map', ','.join(protocols_flavor().service_map())]) if self.grpc_enabled(): args.extend(['-grpc_port', str(self.grpc_port)]) if lameduck_period: args.extend(environment.lameduck_flag(lameduck_period)) if grace_period: args.extend(['-serving_state_grace_period', grace_period]) if security_policy: args.extend(['-security_policy', security_policy]) if extra_args: args.extend(extra_args) args.extend(['-enable-autocommit']) stderr_fd = open( os.path.join(environment.vtlogroot, 'vttablet-%d.stderr' % self.tablet_uid), 'w') # increment count only the first time if not self.proc: Tablet.tablets_running += 1 self.proc = utils.run_bg(args, stderr=stderr_fd, extra_env=extra_env) log_message = ('Started vttablet: %s (%s) with pid: %s - Log files: ' '%s/vttablet.*.{INFO,WARNING,ERROR,FATAL}.*.%s' % (self.tablet_uid, self.tablet_alias, self.proc.pid, environment.vtlogroot, self.proc.pid)) # This may race with the stderr output from the process (though # that's usually empty). stderr_fd.write(log_message + '\n') stderr_fd.close() logging.debug(log_message) # wait for query service to be in the right state if wait_for_state: self.wait_for_vttablet_state(wait_for_state, port=port) if self.tablet_index is not None: topo_server().update_addr('test_' + self.cell, self.keyspace, self.shard, self.tablet_index, (port or self.port)) return self.proc
def start_vttablet( self, port=None, memcache=False, wait_for_state='SERVING', filecustomrules=None, zkcustomrules=None, schema_override=None, repl_extra_flags=None, table_acl_config=None, lameduck_period=None, security_policy=None, full_mycnf_args=False, extra_args=None, extra_env=None, include_mysql_port=True, init_tablet_type=None, init_keyspace=None, init_shard=None, init_db_name_override=None, supports_backups=False, grace_period='1s', enable_semi_sync=True): """Starts a vttablet process, and returns it. The process is also saved in self.proc, so it's easy to kill as well. Returns: the process that was started. """ args = environment.binary_args('vttablet') # Use 'localhost' as hostname because Travis CI worker hostnames # are too long for MySQL replication. args.extend(['-tablet_hostname', 'localhost']) args.extend(['-tablet-path', self.tablet_alias]) args.extend(environment.topo_server().flags()) args.extend(['-binlog_player_protocol', protocols_flavor().binlog_player_protocol()]) args.extend(['-tablet_manager_protocol', protocols_flavor().tablet_manager_protocol()]) args.extend(['-tablet_protocol', protocols_flavor().tabletconn_protocol()]) args.extend(['-binlog_player_healthcheck_topology_refresh', '1s']) args.extend(['-binlog_player_healthcheck_retry_delay', '1s']) args.extend(['-binlog_player_retry_delay', '1s']) args.extend(['-pid_file', os.path.join(self.tablet_dir, 'vttablet.pid')]) # always enable_replication_lag_check with somewhat short values for tests args.extend(['-health_check_interval', '2s']) args.extend(['-enable_replication_lag_check']) args.extend(['-degraded_threshold', '5s']) if enable_semi_sync: args.append('-enable_semi_sync') if self.use_mysqlctld: args.extend( ['-mysqlctl_socket', os.path.join(self.tablet_dir, 'mysqlctl.sock')]) if full_mycnf_args: # this flag is used to specify all the mycnf_ flags, to make # sure that code works. relay_log_path = os.path.join(self.tablet_dir, 'relay-logs', 'vt-%010d-relay-bin' % self.tablet_uid) args.extend([ '-mycnf_server_id', str(self.tablet_uid), '-mycnf_data_dir', os.path.join(self.tablet_dir, 'data'), '-mycnf_innodb_data_home_dir', os.path.join(self.tablet_dir, 'innodb', 'data'), '-mycnf_innodb_log_group_home_dir', os.path.join(self.tablet_dir, 'innodb', 'logs'), '-mycnf_socket_file', os.path.join(self.tablet_dir, 'mysql.sock'), '-mycnf_error_log_path', os.path.join(self.tablet_dir, 'error.log'), '-mycnf_slow_log_path', os.path.join(self.tablet_dir, 'slow-query.log'), '-mycnf_relay_log_path', relay_log_path, '-mycnf_relay_log_index_path', relay_log_path + '.index', '-mycnf_relay_log_info_path', os.path.join(self.tablet_dir, 'relay-logs', 'relay-log.info'), '-mycnf_bin_log_path', os.path.join( self.tablet_dir, 'bin-logs', 'vt-%010d-bin' % self.tablet_uid), '-mycnf_master_info_file', os.path.join(self.tablet_dir, 'master.info'), '-mycnf_pid_file', os.path.join(self.tablet_dir, 'mysql.pid'), '-mycnf_tmp_dir', os.path.join(self.tablet_dir, 'tmp'), '-mycnf_slave_load_tmp_dir', os.path.join(self.tablet_dir, 'tmp'), ]) if include_mysql_port: args.extend(['-mycnf_mysql_port', str(self.mysql_port)]) # this is used to run InitTablet as part of the vttablet startup if init_tablet_type: self.tablet_type = init_tablet_type args.extend(['-init_tablet_type', init_tablet_type]) if init_keyspace: self.keyspace = init_keyspace self.shard = init_shard args.extend(['-init_keyspace', init_keyspace, '-init_shard', init_shard]) if init_db_name_override: self.dbname = init_db_name_override args.extend(['-init_db_name_override', init_db_name_override]) else: self.dbname = 'vt_' + init_keyspace if supports_backups: args.extend(['-restore_from_backup'] + get_backup_storage_flags()) if extra_args: args.extend(extra_args) args.extend(['-port', '%s' % (port or self.port), '-log_dir', environment.vtlogroot]) self._add_dbconfigs(args, repl_extra_flags) if memcache: args.extend(['-rowcache-bin', environment.memcached_bin()]) memcache_socket = os.path.join(self.tablet_dir, 'memcache.sock') args.extend(['-rowcache-socket', memcache_socket]) args.extend(['-enable-rowcache']) if filecustomrules: args.extend(['-filecustomrules', filecustomrules]) if zkcustomrules: args.extend(['-zkcustomrules', zkcustomrules]) if schema_override: args.extend(['-schema-override', schema_override]) if table_acl_config: args.extend(['-table-acl-config', table_acl_config]) args.extend(['-queryserver-config-strict-table-acl']) if protocols_flavor().service_map(): args.extend(['-service_map', ','.join(protocols_flavor().service_map())]) if self.grpc_enabled(): args.extend(['-grpc_port', str(self.grpc_port)]) if lameduck_period: args.extend(['-lameduck-period', lameduck_period]) if grace_period: args.extend(['-serving_state_grace_period', grace_period]) if security_policy: args.extend(['-security_policy', security_policy]) if extra_args: args.extend(extra_args) args.extend(['-enable-autocommit']) stderr_fd = open( os.path.join(environment.vtlogroot, 'vttablet-%d.stderr' % self.tablet_uid), 'w') # increment count only the first time if not self.proc: Tablet.tablets_running += 1 self.proc = utils.run_bg(args, stderr=stderr_fd, extra_env=extra_env) log_message = ( 'Started vttablet: %s (%s) with pid: %s - Log files: ' '%s/vttablet.*.{INFO,WARNING,ERROR,FATAL}.*.%s' % (self.tablet_uid, self.tablet_alias, self.proc.pid, environment.vtlogroot, self.proc.pid)) # This may race with the stderr output from the process (though # that's usually empty). stderr_fd.write(log_message + '\n') stderr_fd.close() logging.debug(log_message) # wait for query service to be in the right state if wait_for_state: self.wait_for_vttablet_state(wait_for_state, port=port) if self.tablet_index is not None: topo_server().update_addr( 'test_'+self.cell, self.keyspace, self.shard, self.tablet_index, (port or self.port)) return self.proc
def start_vttablet( self, port=None, wait_for_state="SERVING", filecustomrules=None, zkcustomrules=None, schema_override=None, repl_extra_flags=None, table_acl_config=None, lameduck_period=None, security_policy=None, full_mycnf_args=False, extra_args=None, extra_env=None, include_mysql_port=True, init_tablet_type=None, init_keyspace=None, init_shard=None, init_db_name_override=None, supports_backups=True, grace_period="1s", enable_semi_sync=True, ): """Starts a vttablet process, and returns it. The process is also saved in self.proc, so it's easy to kill as well. Returns: the process that was started. """ args = environment.binary_args("vttablet") # Use 'localhost' as hostname because Travis CI worker hostnames # are too long for MySQL replication. args.extend(["-tablet_hostname", "localhost"]) args.extend(["-tablet-path", self.tablet_alias]) args.extend(environment.topo_server().flags()) args.extend(["-binlog_player_protocol", protocols_flavor().binlog_player_protocol()]) args.extend(["-tablet_manager_protocol", protocols_flavor().tablet_manager_protocol()]) args.extend(["-tablet_protocol", protocols_flavor().tabletconn_protocol()]) args.extend(["-binlog_player_healthcheck_topology_refresh", "1s"]) args.extend(["-binlog_player_healthcheck_retry_delay", "1s"]) args.extend(["-binlog_player_retry_delay", "1s"]) args.extend(["-pid_file", os.path.join(self.tablet_dir, "vttablet.pid")]) # always enable_replication_reporter with somewhat short values for tests args.extend(["-health_check_interval", "2s"]) args.extend(["-enable_replication_reporter"]) args.extend(["-degraded_threshold", "5s"]) args.extend(["-watch_replication_stream"]) if enable_semi_sync: args.append("-enable_semi_sync") if self.use_mysqlctld: args.extend(["-mysqlctl_socket", os.path.join(self.tablet_dir, "mysqlctl.sock")]) if full_mycnf_args: # this flag is used to specify all the mycnf_ flags, to make # sure that code works. relay_log_path = os.path.join(self.tablet_dir, "relay-logs", "vt-%010d-relay-bin" % self.tablet_uid) args.extend( [ "-mycnf_server_id", str(self.tablet_uid), "-mycnf_data_dir", os.path.join(self.tablet_dir, "data"), "-mycnf_innodb_data_home_dir", os.path.join(self.tablet_dir, "innodb", "data"), "-mycnf_innodb_log_group_home_dir", os.path.join(self.tablet_dir, "innodb", "logs"), "-mycnf_socket_file", os.path.join(self.tablet_dir, "mysql.sock"), "-mycnf_error_log_path", os.path.join(self.tablet_dir, "error.log"), "-mycnf_slow_log_path", os.path.join(self.tablet_dir, "slow-query.log"), "-mycnf_relay_log_path", relay_log_path, "-mycnf_relay_log_index_path", relay_log_path + ".index", "-mycnf_relay_log_info_path", os.path.join(self.tablet_dir, "relay-logs", "relay-log.info"), "-mycnf_bin_log_path", os.path.join(self.tablet_dir, "bin-logs", "vt-%010d-bin" % self.tablet_uid), "-mycnf_master_info_file", os.path.join(self.tablet_dir, "master.info"), "-mycnf_pid_file", os.path.join(self.tablet_dir, "mysql.pid"), "-mycnf_tmp_dir", os.path.join(self.tablet_dir, "tmp"), "-mycnf_slave_load_tmp_dir", os.path.join(self.tablet_dir, "tmp"), ] ) if include_mysql_port: args.extend(["-mycnf_mysql_port", str(self.mysql_port)]) # this is used to run InitTablet as part of the vttablet startup if init_tablet_type: self.tablet_type = init_tablet_type args.extend(["-init_tablet_type", init_tablet_type]) if init_keyspace: self.keyspace = init_keyspace self.shard = init_shard args.extend(["-init_keyspace", init_keyspace, "-init_shard", init_shard]) if init_db_name_override: self.dbname = init_db_name_override args.extend(["-init_db_name_override", init_db_name_override]) else: self.dbname = "vt_" + init_keyspace if supports_backups: args.extend(["-restore_from_backup"] + get_backup_storage_flags()) if extra_args: args.extend(extra_args) args.extend(["-port", "%s" % (port or self.port), "-log_dir", environment.vtlogroot]) self._add_dbconfigs(args, repl_extra_flags) if filecustomrules: args.extend(["-filecustomrules", filecustomrules]) if zkcustomrules: args.extend(["-zkcustomrules", zkcustomrules]) if schema_override: args.extend(["-schema-override", schema_override]) if table_acl_config: args.extend(["-table-acl-config", table_acl_config]) args.extend(["-queryserver-config-strict-table-acl"]) if protocols_flavor().service_map(): args.extend(["-service_map", ",".join(protocols_flavor().service_map())]) if self.grpc_enabled(): args.extend(["-grpc_port", str(self.grpc_port)]) if lameduck_period: args.extend(environment.lameduck_flag(lameduck_period)) if grace_period: args.extend(["-serving_state_grace_period", grace_period]) if security_policy: args.extend(["-security_policy", security_policy]) if extra_args: args.extend(extra_args) args.extend(["-enable-autocommit"]) stderr_fd = open(os.path.join(environment.vtlogroot, "vttablet-%d.stderr" % self.tablet_uid), "w") # increment count only the first time if not self.proc: Tablet.tablets_running += 1 self.proc = utils.run_bg(args, stderr=stderr_fd, extra_env=extra_env) log_message = ( "Started vttablet: %s (%s) with pid: %s - Log files: " "%s/vttablet.*.{INFO,WARNING,ERROR,FATAL}.*.%s" % (self.tablet_uid, self.tablet_alias, self.proc.pid, environment.vtlogroot, self.proc.pid) ) # This may race with the stderr output from the process (though # that's usually empty). stderr_fd.write(log_message + "\n") stderr_fd.close() logging.debug(log_message) # wait for query service to be in the right state if wait_for_state: self.wait_for_vttablet_state(wait_for_state, port=port) if self.tablet_index is not None: topo_server().update_addr( "test_" + self.cell, self.keyspace, self.shard, self.tablet_index, (port or self.port) ) return self.proc
def start_vttablet( self, port=None, wait_for_state='SERVING', topocustomrule_path=None, schema_override=None, repl_extra_flags=None, table_acl_config=None, lameduck_period=None, security_policy=None, full_mycnf_args=False, extra_args=None, extra_env=None, include_mysql_port=True, init_tablet_type=None, init_keyspace=None, init_shard=None, # TODO(mberlin): Assign the index automatically and remove this parameter. tablet_index=None, init_db_name_override=None, binlog_use_v3_resharding_mode=True, supports_backups=True, grace_period='1s', enable_semi_sync=True): # pylint: disable=g-doc-args """Starts a vttablet process, and returns it. The process is also saved in self.proc, so it's easy to kill as well. Returns: the process that was started. """ # pylint: enable=g-doc-args args = environment.binary_args('vttablet') # Use 'localhost' as hostname because Travis CI worker hostnames # are too long for MySQL replication. args.extend(['-tablet_hostname', 'localhost']) args.extend(['-tablet-path', self.tablet_alias]) args.extend(environment.topo_server().flags()) args.extend([ '-binlog_player_protocol', protocols_flavor().binlog_player_protocol() ]) args.extend([ '-tablet_manager_protocol', protocols_flavor().tablet_manager_protocol() ]) args.extend( ['-tablet_protocol', protocols_flavor().tabletconn_protocol()]) args.extend(['-vreplication_healthcheck_topology_refresh', '1s']) args.extend(['-vreplication_healthcheck_retry_delay', '1s']) args.extend(['-vreplication_retry_delay', '1s']) args.extend( ['-pid_file', os.path.join(self.tablet_dir, 'vttablet.pid')]) # always enable_replication_reporter with somewhat short values for tests args.extend(['-health_check_interval', '2s']) args.extend(['-enable_replication_reporter']) args.extend(['-degraded_threshold', '5s']) args.extend(['-lock_tables_timeout', '5s']) args.extend(['-watch_replication_stream']) if enable_semi_sync: args.append('-enable_semi_sync') # Remember the setting in case a test wants to know it. self.enable_semi_sync = enable_semi_sync if self.use_mysqlctld: args.extend([ '-mysqlctl_socket', os.path.join(self.tablet_dir, 'mysqlctl.sock') ]) if self.external_mysql: args.extend(['-db_host', '127.0.0.1']) args.extend(['-db_port', str(self.mysql_port)]) args.append('-disable_active_reparents') if full_mycnf_args: # this flag is used to specify all the mycnf_ flags, to make # sure that code works. relay_log_path = os.path.join( self.tablet_dir, 'relay-logs', 'vt-%010d-relay-bin' % self.tablet_uid) args.extend([ '-mycnf_server_id', str(self.tablet_uid), '-mycnf_data_dir', os.path.join(self.tablet_dir, 'data'), '-mycnf_innodb_data_home_dir', os.path.join(self.tablet_dir, 'innodb', 'data'), '-mycnf_innodb_log_group_home_dir', os.path.join(self.tablet_dir, 'innodb', 'logs'), '-mycnf_socket_file', os.path.join(self.tablet_dir, 'mysql.sock'), '-mycnf_error_log_path', os.path.join(self.tablet_dir, 'error.log'), '-mycnf_slow_log_path', os.path.join(self.tablet_dir, 'slow-query.log'), '-mycnf_relay_log_path', relay_log_path, '-mycnf_relay_log_index_path', relay_log_path + '.index', '-mycnf_relay_log_info_path', os.path.join(self.tablet_dir, 'relay-logs', 'relay-log.info'), '-mycnf_bin_log_path', os.path.join(self.tablet_dir, 'bin-logs', 'vt-%010d-bin' % self.tablet_uid), '-mycnf_master_info_file', os.path.join(self.tablet_dir, 'master.info'), '-mycnf_pid_file', os.path.join(self.tablet_dir, 'mysql.pid'), '-mycnf_tmp_dir', os.path.join(self.tablet_dir, 'tmp'), '-mycnf_slave_load_tmp_dir', os.path.join(self.tablet_dir, 'tmp'), ]) if include_mysql_port: args.extend(['-mycnf_mysql_port', str(self.mysql_port)]) # this is used to run InitTablet as part of the vttablet startup if init_tablet_type: self.tablet_type = init_tablet_type args.extend(['-init_tablet_type', init_tablet_type]) if init_keyspace: self.keyspace = init_keyspace self.shard = init_shard # tablet_index is required for the update_addr call below. if self.tablet_index is None: self.tablet_index = tablet_index args.extend( ['-init_keyspace', init_keyspace, '-init_shard', init_shard]) if init_db_name_override: self.dbname = init_db_name_override args.extend(['-init_db_name_override', init_db_name_override]) else: self.dbname = 'vt_' + init_keyspace # Default value for this flag is True. So, add it only if it's false. if not binlog_use_v3_resharding_mode: args.extend(['-binlog_use_v3_resharding_mode=false']) if supports_backups: args.extend(['-restore_from_backup'] + get_backup_storage_flags()) # When vttablet restores from backup, it will re-generate the .cnf file. # So we need to have EXTRA_MY_CNF set properly. # When using mysqlctld, only mysqlctld should need EXTRA_MY_CNF. # If any test fails without giving EXTRA_MY_CNF to vttablet, # it means we missed some call that should run remotely on mysqlctld. if not self.use_mysqlctld: all_extra_my_cnf = get_all_extra_my_cnf(None) if all_extra_my_cnf: if not extra_env: extra_env = {} extra_env['EXTRA_MY_CNF'] = ':'.join(all_extra_my_cnf) if extra_args: args.extend(extra_args) args.extend([ '-port', '%s' % (port or self.port), '-log_dir', environment.vtlogroot ]) if topocustomrule_path: args.extend(['-topocustomrule_path', topocustomrule_path]) if schema_override: args.extend(['-schema-override', schema_override]) if table_acl_config: args.extend(['-table-acl-config', table_acl_config]) args.extend(['-queryserver-config-strict-table-acl']) if protocols_flavor().service_map(): args.extend( ['-service_map', ','.join(protocols_flavor().service_map())]) if self.grpc_enabled(): args.extend(['-grpc_port', str(self.grpc_port)]) args.extend([ '-grpc_max_message_size', str(environment.grpc_max_message_size) ]) if lameduck_period: args.extend(environment.lameduck_flag(lameduck_period)) if grace_period: args.extend(['-serving_state_grace_period', grace_period]) if security_policy: args.extend(['-security_policy', security_policy]) args.extend(['-enable-autocommit']) stderr_fd = open( os.path.join(environment.vtlogroot, 'vttablet-%d.stderr' % self.tablet_uid), 'w') # increment count only the first time if not self.proc: Tablet.tablets_running += 1 self.proc = utils.run_bg(args, stderr=stderr_fd, extra_env=extra_env) log_message = ('Started vttablet: %s (%s) with pid: %s - Log files: ' '%s/vttablet.*.{INFO,WARNING,ERROR,FATAL}.*.%s' % (self.tablet_uid, self.tablet_alias, self.proc.pid, environment.vtlogroot, self.proc.pid)) # This may race with the stderr output from the process (though # that's usually empty). stderr_fd.write(log_message + '\n') stderr_fd.close() logging.debug(log_message) # wait for query service to be in the right state if wait_for_state: self.wait_for_vttablet_state(wait_for_state, port=port) if self.tablet_index is not None: topo_server().update_addr('test_' + self.cell, self.keyspace, self.shard, self.tablet_index, (port or self.port)) return self.proc
def start_vttablet( self, port=None, wait_for_state='SERVING', topocustomrule_path=None, schema_override=None, repl_extra_flags=None, table_acl_config=None, lameduck_period=None, security_policy=None, full_mycnf_args=False, extra_args=None, extra_env=None, include_mysql_port=True, init_tablet_type=None, init_keyspace=None, init_shard=None, # TODO(mberlin): Assign the index automatically and remove this parameter. tablet_index=None, init_db_name_override=None, binlog_use_v3_resharding_mode=True, supports_backups=True, grace_period='1s', enable_semi_sync=True): # pylint: disable=g-doc-args """Starts a vttablet process, and returns it. The process is also saved in self.proc, so it's easy to kill as well. Returns: the process that was started. """ # pylint: enable=g-doc-args args = environment.binary_args('vttablet') # Use 'localhost' as hostname because Travis CI worker hostnames # are too long for MySQL replication. args.extend(['-tablet_hostname', 'localhost']) args.extend(['-tablet-path', self.tablet_alias]) args.extend(environment.topo_server().flags()) args.extend(['-binlog_player_protocol', protocols_flavor().binlog_player_protocol()]) args.extend(['-tablet_manager_protocol', protocols_flavor().tablet_manager_protocol()]) args.extend(['-tablet_protocol', protocols_flavor().tabletconn_protocol()]) args.extend(['-binlog_player_healthcheck_topology_refresh', '1s']) args.extend(['-binlog_player_healthcheck_retry_delay', '1s']) args.extend(['-binlog_player_retry_delay', '1s']) args.extend(['-pid_file', os.path.join(self.tablet_dir, 'vttablet.pid')]) # always enable_replication_reporter with somewhat short values for tests args.extend(['-health_check_interval', '2s']) args.extend(['-enable_replication_reporter']) args.extend(['-degraded_threshold', '5s']) args.extend(['-watch_replication_stream']) if enable_semi_sync: args.append('-enable_semi_sync') # Remember the setting in case a test wants to know it. self.enable_semi_sync = enable_semi_sync if self.use_mysqlctld: args.extend( ['-mysqlctl_socket', os.path.join(self.tablet_dir, 'mysqlctl.sock')]) if full_mycnf_args: # this flag is used to specify all the mycnf_ flags, to make # sure that code works. relay_log_path = os.path.join(self.tablet_dir, 'relay-logs', 'vt-%010d-relay-bin' % self.tablet_uid) args.extend([ '-mycnf_server_id', str(self.tablet_uid), '-mycnf_data_dir', os.path.join(self.tablet_dir, 'data'), '-mycnf_innodb_data_home_dir', os.path.join(self.tablet_dir, 'innodb', 'data'), '-mycnf_innodb_log_group_home_dir', os.path.join(self.tablet_dir, 'innodb', 'logs'), '-mycnf_socket_file', os.path.join(self.tablet_dir, 'mysql.sock'), '-mycnf_error_log_path', os.path.join(self.tablet_dir, 'error.log'), '-mycnf_slow_log_path', os.path.join(self.tablet_dir, 'slow-query.log'), '-mycnf_relay_log_path', relay_log_path, '-mycnf_relay_log_index_path', relay_log_path + '.index', '-mycnf_relay_log_info_path', os.path.join(self.tablet_dir, 'relay-logs', 'relay-log.info'), '-mycnf_bin_log_path', os.path.join( self.tablet_dir, 'bin-logs', 'vt-%010d-bin' % self.tablet_uid), '-mycnf_master_info_file', os.path.join(self.tablet_dir, 'master.info'), '-mycnf_pid_file', os.path.join(self.tablet_dir, 'mysql.pid'), '-mycnf_tmp_dir', os.path.join(self.tablet_dir, 'tmp'), '-mycnf_slave_load_tmp_dir', os.path.join(self.tablet_dir, 'tmp'), ]) if include_mysql_port: args.extend(['-mycnf_mysql_port', str(self.mysql_port)]) # this is used to run InitTablet as part of the vttablet startup if init_tablet_type: self.tablet_type = init_tablet_type args.extend(['-init_tablet_type', init_tablet_type]) if init_keyspace: self.keyspace = init_keyspace self.shard = init_shard # tablet_index is required for the update_addr call below. if self.tablet_index is None: self.tablet_index = tablet_index args.extend(['-init_keyspace', init_keyspace, '-init_shard', init_shard]) if init_db_name_override: self.dbname = init_db_name_override args.extend(['-init_db_name_override', init_db_name_override]) else: self.dbname = 'vt_' + init_keyspace # Default value for this flag is True. So, add it only if it's false. if not binlog_use_v3_resharding_mode: args.extend(['-binlog_use_v3_resharding_mode=false']) if supports_backups: args.extend(['-restore_from_backup'] + get_backup_storage_flags()) # When vttablet restores from backup, it will re-generate the .cnf file. # So we need to have EXTRA_MY_CNF set properly. all_extra_my_cnf = get_all_extra_my_cnf(None) if all_extra_my_cnf: if not extra_env: extra_env = {} extra_env['EXTRA_MY_CNF'] = ':'.join(all_extra_my_cnf) if extra_args: args.extend(extra_args) args.extend(['-port', '%s' % (port or self.port), '-log_dir', environment.vtlogroot]) self._add_dbconfigs(self.default_db_config, args, repl_extra_flags) if topocustomrule_path: args.extend(['-topocustomrule_path', topocustomrule_path]) if schema_override: args.extend(['-schema-override', schema_override]) if table_acl_config: args.extend(['-table-acl-config', table_acl_config]) args.extend(['-queryserver-config-strict-table-acl']) if protocols_flavor().service_map(): args.extend(['-service_map', ','.join(protocols_flavor().service_map())]) if self.grpc_enabled(): args.extend(['-grpc_port', str(self.grpc_port)]) args.extend(['-grpc_max_message_size', str(environment.grpc_max_message_size)]) if lameduck_period: args.extend(environment.lameduck_flag(lameduck_period)) if grace_period: args.extend(['-serving_state_grace_period', grace_period]) if security_policy: args.extend(['-security_policy', security_policy]) args.extend(['-enable-autocommit']) stderr_fd = open( os.path.join(environment.vtlogroot, 'vttablet-%d.stderr' % self.tablet_uid), 'w') # increment count only the first time if not self.proc: Tablet.tablets_running += 1 self.proc = utils.run_bg(args, stderr=stderr_fd, extra_env=extra_env) log_message = ( 'Started vttablet: %s (%s) with pid: %s - Log files: ' '%s/vttablet.*.{INFO,WARNING,ERROR,FATAL}.*.%s' % (self.tablet_uid, self.tablet_alias, self.proc.pid, environment.vtlogroot, self.proc.pid)) # This may race with the stderr output from the process (though # that's usually empty). stderr_fd.write(log_message + '\n') stderr_fd.close() logging.debug(log_message) # wait for query service to be in the right state if wait_for_state: self.wait_for_vttablet_state(wait_for_state, port=port) if self.tablet_index is not None: topo_server().update_addr( 'test_'+self.cell, self.keyspace, self.shard, self.tablet_index, (port or self.port)) return self.proc