def wait_until_stopped(self, wait_seconds=127, marks=[], dump_core=True):
"""
Wait until node is stopped after do_stop was called.
- wait_other_notice: return only when the other live nodes of the
cluster have marked this node has dead.
- marks: optional list of (node, mark) to call watch_log_for_death on.
"""
if self.is_running():
if not self._wait_until_stopped(wait_seconds):
if self.jmx_pid:
try:
os.kill(self.jmx_pid, signal.SIGKILL)
except OSError:
pass
if dump_core and self.pid:
# Aborting is intended to generate a core dump
# so the reason the node didn't stop normally can be studied.
print(
"{} is still running. Trying to generate coredump using kill({}, SIGQUIT)..."
.format(self.name, self.pid))
try:
os.kill(self.pid, signal.SIGQUIT)
except OSError:
pass
self._wait_until_stopped(300)
if self.is_running():
raise NodeError("Problem stopping node %s" % self.name)
for node, mark in marks:
if node != self:
node.watch_log_for_death(self, from_mark=mark)
def _update_pid(self):
if not os.path.isfile(self._get_pid_file()):
return
start = time.time()
pidfile = self._get_pid_file()
while not (os.path.isfile(pidfile) and os.stat(pidfile).st_size > 0):
if time.time() - start > 30.0:
print_(
"Timed out waiting for pidfile {} to be filled (current time is %s): File {} size={}"
.format(
pidfile, datetime.now(), 'exists'
if os.path.isfile(pidfile) else 'does not exist'
if not os.path.exists(pidfile) else 'is not a file',
os.stat(pidfile).st_size
if os.path.exists(pidfile) else -1))
break
else:
time.sleep(0.1)
try:
with open(self._get_pid_file(), 'r') as f:
self._pid = int(f.readline().strip())
except IOError as e:
raise NodeError('Problem starting scylla-manager due to %s' % (e))
def replace_nonexistent_node_test(self):
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
[node1, node2, node3] = cluster.nodelist()
debug("Inserting Data...")
if cluster.version() < "2.1":
node1.stress(
['-o', 'insert', '--num-keys=10000', '--replication-factor=3'])
else:
node1.stress(
['write', 'n=10000', '-schema', 'replication(factor=3)'])
cursor = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1' if self.cluster.version(
) >= '2.1' else '"Keyspace1"."Standard1"'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table,
consistency_level=ConsistencyLevel.THREE)
initialData = cursor.execute(query)
debug('Start node 4 and replace an address with no node')
node4 = Node('node4', cluster, True, ('127.0.0.4', 9160),
('127.0.0.4', 7000), '7400', '0', None,
('127.0.0.4', 9042))
cluster.add(node4, False)
#try to replace an unassigned ip address
with self.assertRaises(NodeError):
try:
node4.start(replace_address='127.0.0.5',
wait_for_binary_proto=True)
except (NodeError, TimeoutError):
raise NodeError("Node could not start.")
def _update_jmx_pid(self, wait=True):
pidfile = os.path.join(self.get_path(), 'scylla-jmx.pid')
start = time.time()
while not (os.path.isfile(pidfile) and os.stat(pidfile).st_size > 0):
elapsed = time.time() - start
if elapsed > 30.0 or not wait:
if wait:
print_(
"Timed out waiting for pidfile {} to be filled (after {} seconds): File {} size={}"
.format(
pidfile, elapsed, 'exists'
if os.path.isfile(pidfile) else 'does not exist' if
not os.path.exists(pidfile) else 'is not a file',
os.stat(pidfile).st_size
if os.path.exists(pidfile) else -1))
break
else:
time.sleep(0.1)
if not wait:
self.jmx_pid = None
return
try:
with open(pidfile, 'r') as f:
self.jmx_pid = int(f.readline().strip())
except IOError as e:
raise NodeError('Problem starting node %s scylla-jmx due to %s' %
(self.name, e))
def start_nodes(self,
nodes=None,
no_wait=False,
verbose=False,
wait_for_binary_proto=False,
wait_other_notice=False,
jvm_args=None,
profile_options=None,
quiet_start=False):
if not self.started and self.force_wait_for_cluster_start:
wait_other_notice = True
wait_for_binary_proto = True
self.started = True
p = None
if jvm_args is None:
jvm_args = []
marks = []
if wait_other_notice:
marks = [(node, node.mark_log()) for node in self.nodes.values()
if node.is_running()]
if nodes is None:
nodes = self.nodes.values()
elif isinstance(nodes, ScyllaNode):
nodes = [nodes]
started = []
for node in nodes:
if not node.is_running():
mark = 0
if os.path.exists(node.logfilename()):
mark = node.mark_log()
p = node.start(update_pid=False,
jvm_args=jvm_args,
profile_options=profile_options)
started.append((node, p, mark))
self.__update_pids(started)
for node, p, _ in started:
if not node.is_running():
raise NodeError("Error starting {0}.".format(node.name), p)
if wait_for_binary_proto:
for node, _, mark in started:
node.watch_log_for("Starting listening for CQL clients",
verbose=verbose,
from_mark=mark)
if wait_other_notice:
for old_node, mark in marks:
for node, _, _ in started:
if old_node is not node:
old_node.watch_log_for_alive(node, from_mark=mark)
return started
def _start_scylla(self, args, marks, update_pid, wait_other_notice,
wait_for_binary_proto):
log_file = os.path.join(self.get_path(), 'logs', 'system.log')
# In case we are restarting a node
# we risk reading the old cassandra.pid file
self._delete_old_pid()
scylla_log = open(log_file, 'a')
env_copy = os.environ
env_copy['SCYLLA_HOME'] = self.get_path()
dbuild_so_dir = os.environ.get('SCYLLA_DBUILD_SO_DIR')
if dbuild_so_dir:
# FIXME: this should be removed once we'll support running scylla relocatable package
executable = os.path.join(dbuild_so_dir, 'scylla.sh')
if not os.path.isfile(executable):
with open(executable, 'w+') as f:
f.write('#!/bin/bash\nexec -a scylla ' + ' '.join([
os.path.join(dbuild_so_dir, 'ld-linux-x86-64.so.2'),
'--library-path', dbuild_so_dir
]) + ' "$@" ')
os.chmod(executable, 0o0777)
args = [executable] + args
self._process_scylla = subprocess.Popen(args,
stdout=scylla_log,
stderr=scylla_log,
close_fds=True,
env=env_copy)
self._process_scylla.poll()
# When running on ccm standalone, the waiter thread would block
# the create commands. Besides in that mode, waiting is unnecessary,
# since the original popen reference is garbage collected.
standalone = os.environ.get('SCYLLA_CCM_STANDALONE', None)
if standalone is None:
self._process_scylla_waiter = threading.Thread(
target=self._wait_for_scylla)
self._process_scylla_waiter.start()
pid_filename = os.path.join(self.get_path(), 'cassandra.pid')
with open(pid_filename, 'w') as pid_file:
pid_file.write(str(self._process_scylla.pid))
if update_pid:
self._update_pid(self._process_scylla)
if not self.is_running():
raise NodeError("Error starting node %s" % self.name,
self._process_scylla)
if wait_other_notice:
for node, mark in marks:
node.watch_log_for_alive(self, from_mark=mark)
if wait_for_binary_proto:
self.wait_for_binary_interface(from_mark=self.mark,
process=self._process_scylla)
else:
time.sleep(2)
return self._process_scylla
def _start_scylla(self, args, marks, update_pid, wait_other_notice,
wait_for_binary_proto, ext_env):
log_file = os.path.join(self.get_path(), 'logs', 'system.log')
# In case we are restarting a node
# we risk reading the old cassandra.pid file
self._delete_old_pid()
scylla_log = open(log_file, 'a')
try:
env_copy = self._launch_env
except AttributeError:
env_copy = os.environ
env_copy['SCYLLA_HOME'] = self.get_path()
env_copy.update(ext_env)
self._process_scylla = subprocess.Popen(args,
stdout=scylla_log,
stderr=scylla_log,
close_fds=True,
env=env_copy)
self._process_scylla.poll()
# When running on ccm standalone, the waiter thread would block
# the create commands. Besides in that mode, waiting is unnecessary,
# since the original popen reference is garbage collected.
standalone = os.environ.get('SCYLLA_CCM_STANDALONE', None)
if standalone is None:
self._process_scylla_waiter = threading.Thread(
target=self._wait_for_scylla)
# Don't block the main thread on abnormal shutdown
self._process_scylla_waiter.daemon = True
self._process_scylla_waiter.start()
pid_filename = os.path.join(self.get_path(), 'cassandra.pid')
with open(pid_filename, 'w') as pid_file:
pid_file.write(str(self._process_scylla.pid))
if update_pid:
self._update_pid(self._process_scylla)
if not self.is_running():
raise NodeError("Error starting node %s" % self.name,
self._process_scylla)
if wait_other_notice:
for node, mark in marks:
node.watch_log_for_alive(self, from_mark=mark)
if wait_for_binary_proto:
try:
self.wait_for_binary_interface(from_mark=self.mark,
process=self._process_scylla,
timeout=420)
except TimeoutError as e:
if not self.wait_for_bootstrap_repair(from_mark=self.mark):
raise e
pass
return self._process_scylla
def start(self,
no_wait=False,
verbose=False,
wait_for_binary_proto=False,
jvm_args=[],
profile_options=None):
started = []
for node in list(self.nodes.values()):
if not node.is_running():
mark = 0
if os.path.exists(node.logfilename()):
mark = node.mark_log()
p = node.start(update_pid=False,
jvm_args=jvm_args,
profile_options=profile_options)
started.append((node, p, mark))
if no_wait and not verbose:
time.sleep(
2
) # waiting 2 seconds to check for early errors and for the pid to be set
else:
for node, p, mark in started:
try:
node.watch_log_for("Listening for thrift clients...",
process=p,
verbose=verbose,
from_mark=mark)
except RuntimeError:
return None
self.__update_pids(started)
for node, p, _ in started:
if not node.is_running():
raise NodeError("Error starting {0}.".format(node.name), p)
if not no_wait and self.version() >= "0.8":
# 0.7 gossip messages seems less predictible that from 0.8 onwards and
# I don't care enough
for node, _, mark in started:
for other_node, _, _ in started:
if other_node is not node:
node.watch_log_for_alive(other_node, from_mark=mark)
if wait_for_binary_proto:
for node, _, mark in started:
node.watch_log_for("Starting listening for CQL clients",
process=p,
verbose=verbose,
from_mark=mark)
time.sleep(0.2)
return started
def start(self, no_wait=False, verbose=False, wait_for_binary_proto=False, wait_other_notice=False, jvm_args=None, profile_options=None, quiet_start=False):
if jvm_args is None:
jvm_args = []
common.assert_jdk_valid_for_cassandra_version(self.cassandra_version())
if wait_other_notice:
marks = [(node, node.mark_log()) for node in list(self.nodes.values())]
started = []
for node in list(self.nodes.values()):
if not node.is_running():
mark = 0
if os.path.exists(node.logfilename()):
mark = node.mark_log()
p = node.start(update_pid=False, jvm_args=jvm_args, profile_options=profile_options, verbose=verbose, quiet_start=quiet_start)
started.append((node, p, mark))
if no_wait and not verbose:
time.sleep(2) # waiting 2 seconds to check for early errors and for the pid to be set
else:
for node, p, mark in started:
try:
start_message = "Listening for thrift clients..." if parse_version(self.version()) < parse_version("2.2") else "Starting listening for CQL clients"
node.watch_log_for(start_message, timeout=60, process=p, verbose=verbose, from_mark=mark)
except RuntimeError:
return None
self.__update_pids(started)
for node, p, _ in started:
if not node.is_running():
raise NodeError("Error starting {0}.".format(node.name), p)
if not no_wait and parse_version(self.version()) >= parse_version("0.8"):
# 0.7 gossip messages seems less predictible that from 0.8 onwards and
# I don't care enough
for node, _, mark in started:
for other_node, _, _ in started:
if other_node is not node:
node.watch_log_for_alive(other_node, from_mark=mark)
if wait_other_notice:
for old_node, mark in marks:
for node, _, _ in started:
if old_node is not node:
old_node.watch_log_for_alive(node, from_mark=mark)
if wait_for_binary_proto:
for node, p, mark in started:
node.wait_for_binary_interface(process=p, verbose=verbose, from_mark=mark)
return started
def _wait_no_pending_flushes(self, wait_timeout=60):
def no_pending_flushes():
stdout, _ = self.nodetool('cfstats')
pending_flushes = False
for line in stdout.splitlines():
line = line.strip()
if line.startswith('Pending flushes'):
_, pending_flushes_str = line.split(':')
pending_flushes_count = int(pending_flushes_str.strip())
if pending_flushes_count > 0:
pending_flushes = True
return not pending_flushes
result = wait_for(no_pending_flushes, timeout=wait_timeout, step=1.0)
if result is None:
raise NodeError("Node %s still has pending flushes after "
"%s seconds" % (self.name, wait_timeout))
def _update_jmx_pid(self):
pidfile = os.path.join(self.get_path(), 'scylla-jmx.pid')
start = time.time()
while not (os.path.isfile(pidfile) and os.stat(pidfile).st_size > 0):
if time.time() - start > 30.0:
print_("Timed out waiting for pidfile to be filled "
"(current time is %s)" % (datetime.datetime.now()))
break
else:
time.sleep(0.1)
try:
with open(pidfile, 'r') as f:
self.jmx_pid = int(f.readline().strip())
except IOError as e:
raise NodeError('Problem starting node %s scylla-jmx due to %s' %
(self.name, e))
def _update_pid(self):
if not os.path.isfile(self._get_pid_file()):
return
start = time.time()
while not (os.path.isfile(self._get_pid_file())
and os.stat(self._get_pid_file()).st_size > 0):
if time.time() - start > 30.0:
print_("Timed out waiting for pidfile to be filled "
"(current time is %s)" % (datetime.datetime.now()))
break
else:
time.sleep(0.1)
try:
with open(self._get_pid_file(), 'r') as f:
self._pid = int(f.readline().strip())
except IOError as e:
raise NodeError('Problem starting scylla-manager due to %s' % (e))
def replace_active_node_test(self):
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
[node1, node2, node3] = cluster.nodelist()
debug("Inserting Data...")
if cluster.version() < "2.1":
node1.stress(
['-o', 'insert', '--num-keys=10000', '--replication-factor=3'])
else:
node1.stress(
['write', 'n=10000', '-schema', 'replication(factor=3)'])
cursor = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1' if self.cluster.version(
) >= '2.1' else '"Keyspace1"."Standard1"'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table,
consistency_level=ConsistencyLevel.THREE)
initialData = cursor.execute(query)
#replace active node 3 with node 4
debug("Starting node 4 to replace active node 3")
node4 = Node('node4', cluster, True, ('127.0.0.4', 9160),
('127.0.0.4', 7000), '7400', '0', None,
('127.0.0.4', 9042))
cluster.add(node4, False)
with self.assertRaises(NodeError):
try:
node4.start(replace_address='127.0.0.3',
wait_for_binary_proto=True)
except (NodeError, TimeoutError):
raise NodeError("Node could not start.")
checkError = node4.grep_log(
"java.lang.UnsupportedOperationException: Cannot replace a live node..."
)
self.assertEqual(len(checkError), 1)
def start(self, no_wait=False, verbose=False, wait_for_binary_proto=False,
wait_other_notice=True, jvm_args=None, profile_options=None,
quiet_start=False, allow_root=False):
if jvm_args is None:
jvm_args = []
common.assert_jdk_valid_for_cassandra_version(self.cassandra_version())
# check whether all loopback aliases are available before starting any nodes
for node in list(self.nodes.values()):
if not node.is_running():
for itf in node.network_interfaces.values():
if itf is not None:
if not common.check_socket_available(itf, return_on_error=True):
addr, port = itf
common.error("Inet address {}:{} is not available; a cluster may already be running or you may need to add the loopback alias".format(addr, port))
sys.exit(1)
started = []
for node in list(self.nodes.values()):
if not node.is_running():
mark = 0
if os.path.exists(node.logfilename()):
mark = node.mark_log()
p = node.start(update_pid=False, jvm_args=jvm_args, profile_options=profile_options, verbose=verbose, quiet_start=quiet_start, allow_root=allow_root)
# Prior to JDK8, starting every node at once could lead to a
# nanotime collision where the RNG that generates a node's tokens
# gives identical tokens to several nodes. Thus, we stagger
# the node starts
if common.get_jdk_version() < '1.8':
time.sleep(1)
started.append((node, p, mark))
if no_wait:
time.sleep(2) # waiting 2 seconds to check for early errors and for the pid to be set
else:
for node, p, mark in started:
try:
start_message = "Listening for thrift clients..." if self.cassandra_version() < "2.2" else "Starting listening for CQL clients"
node.watch_log_for(start_message, timeout=60, process=p, verbose=verbose, from_mark=mark)
except RuntimeError:
return None
self.__update_pids(started)
for node, p, _ in started:
if not node.is_running():
raise NodeError("Error starting {0}.".format(node.name), p)
if not no_wait:
if wait_other_notice:
for (node, _, mark), (other_node, _, _) in itertools.permutations(started, 2):
node.watch_log_for_alive(other_node, from_mark=mark)
if wait_for_binary_proto:
for node, p, mark in started:
node.wait_for_binary_interface(process=p, verbose=verbose, from_mark=mark)
return started
def start(self,
no_wait=False,
verbose=False,
wait_for_binary_proto=True,
wait_other_notice=True,
jvm_args=None,
profile_options=None,
quiet_start=False,
allow_root=False,
jvm_version=None,
**kwargs):
if jvm_args is None:
jvm_args = []
extension.pre_cluster_start(self)
# check whether all loopback aliases are available before starting any nodes
for node in list(self.nodes.values()):
if not node.is_running():
for itf in node.network_interfaces.values():
if itf is not None:
common.assert_socket_available(itf)
started = []
for node in list(self.nodes.values()):
if not node.is_running():
mark = 0
if os.path.exists(node.logfilename()):
mark = node.mark_log()
p = node.start(update_pid=False,
jvm_args=jvm_args,
jvm_version=jvm_version,
profile_options=profile_options,
verbose=verbose,
quiet_start=quiet_start,
allow_root=allow_root)
# Prior to JDK8, starting every node at once could lead to a
# nanotime collision where the RNG that generates a node's tokens
# gives identical tokens to several nodes. Thus, we stagger
# the node starts
if common.get_jdk_version() < '1.8':
time.sleep(1)
started.append((node, p, mark))
if no_wait:
time.sleep(
2
) # waiting 2 seconds to check for early errors and for the pid to be set
else:
for node, p, mark in started:
try:
timeout = kwargs.get(
'timeout',
int(
os.environ.get('CCM_CLUSTER_START_DEFAULT_TIMEOUT',
60)))
timeout = int(
os.environ.get('CCM_CLUSTER_START_TIMEOUT_OVERRIDE',
timeout))
start_message = "Listening for thrift clients..." if self.cassandra_version(
) < "2.2" else "Starting listening for CQL clients"
node.watch_log_for(start_message,
timeout=timeout,
process=p,
verbose=verbose,
from_mark=mark)
except RuntimeError:
return None
self.__update_pids(started)
for node, p, _ in started:
if not node.is_running():
raise NodeError("Error starting {0}.".format(node.name), p)
if not no_wait:
if wait_other_notice:
for (node, _, mark), (other_node, _,
_) in itertools.permutations(started, 2):
node.watch_log_for_alive(other_node, from_mark=mark)
if wait_for_binary_proto:
for node, p, mark in started:
node.wait_for_binary_interface(process=p,
verbose=verbose,
from_mark=mark)
extension.post_cluster_start(self)
return started
def start(self,
join_ring=True,
no_wait=False,
verbose=False,
update_pid=True,
wait_other_notice=False,
replace_token=None,
replace_address=None,
jvm_args=None,
wait_for_binary_proto=False,
profile_options=None,
use_jna=False,
quiet_start=False):
"""
Start the node. Options includes:
- join_ring: if false, start the node with -Dcassandra.join_ring=False
- no_wait: by default, this method returns when the node is started and listening to clients.
If no_wait=True, the method returns sooner.
- wait_other_notice: if True, this method returns only when all other live node of the cluster
have marked this node UP.
- replace_token: start the node with the -Dcassandra.replace_token option.
- replace_address: start the node with the -Dcassandra.replace_address option.
"""
if jvm_args is None:
jvm_args = []
if self.is_running():
raise NodeError("%s is already running" % self.name)
for itf in list(self.network_interfaces.values()):
if itf is not None and replace_address is None:
common.check_socket_available(itf)
if wait_other_notice:
marks = [(node, node.mark_log())
for node in list(self.cluster.nodes.values())
if node.is_running()]
cdir = self.get_install_dir()
launch_bin = common.join_bin(cdir, 'bin', 'dse')
# Copy back the dse scripts since profiling may have modified it the previous time
shutil.copy(launch_bin, self.get_bin_dir())
launch_bin = common.join_bin(self.get_path(), 'bin', 'dse')
# If Windows, change entries in .bat file to split conf from binaries
if common.is_win():
self.__clean_bat()
if profile_options is not None:
config = common.get_config()
if 'yourkit_agent' not in config:
raise NodeError(
"Cannot enable profile. You need to set 'yourkit_agent' to the path of your agent in a {0}/config"
.format(common.get_default_path_display_name()))
cmd = '-agentpath:%s' % config['yourkit_agent']
if 'options' in profile_options:
cmd = cmd + '=' + profile_options['options']
print_(cmd)
# Yes, it's fragile as shit
pattern = r'cassandra_parms="-Dlog4j.configuration=log4j-server.properties -Dlog4j.defaultInitOverride=true'
common.replace_in_file(launch_bin, pattern,
' ' + pattern + ' ' + cmd + '"')
os.chmod(launch_bin, os.stat(launch_bin).st_mode | stat.S_IEXEC)
env = common.make_dse_env(self.get_install_dir(), self.get_path())
if common.is_win():
self._clean_win_jmx()
pidfile = os.path.join(self.get_path(), 'cassandra.pid')
args = [launch_bin, 'cassandra']
if self.workload is not None:
if 'hadoop' in self.workload:
args.append('-t')
if 'solr' in self.workload:
args.append('-s')
if 'spark' in self.workload:
args.append('-k')
if 'cfs' in self.workload:
args.append('-c')
args += ['-p', pidfile, '-Dcassandra.join_ring=%s' % str(join_ring)]
if replace_token is not None:
args.append('-Dcassandra.replace_token=%s' % str(replace_token))
if replace_address is not None:
args.append('-Dcassandra.replace_address=%s' %
str(replace_address))
if use_jna is False:
args.append('-Dcassandra.boot_without_jna=true')
args = args + jvm_args
process = None
if common.is_win():
# clean up any old dirty_pid files from prior runs
if (os.path.isfile(self.get_path() + "/dirty_pid.tmp")):
os.remove(self.get_path() + "/dirty_pid.tmp")
process = subprocess.Popen(args,
cwd=self.get_bin_dir(),
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
else:
process = subprocess.Popen(args,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# Our modified batch file writes a dirty output with more than just the pid - clean it to get in parity
# with *nix operation here.
if common.is_win():
self.__clean_win_pid()
self._update_pid(process)
elif update_pid:
if no_wait:
time.sleep(
2
) # waiting 2 seconds nevertheless to check for early errors and for the pid to be set
else:
for line in process.stdout:
if verbose:
print_(line.rstrip('\n'))
self._update_pid(process)
if not self.is_running():
raise NodeError("Error starting node %s" % self.name, process)
if wait_other_notice:
for node, mark in marks:
node.watch_log_for_alive(self, from_mark=mark)
if wait_for_binary_proto:
self.wait_for_binary_interface()
if self.cluster.hasOpscenter():
self._start_agent()
return process
def stop(self,
wait=True,
wait_other_notice=False,
other_nodes=None,
gently=True,
wait_seconds=127):
"""
Stop the node.
- wait: if True (the default), wait for the Scylla process to be
really dead. Otherwise return after having sent the kill signal.
stop() will wait up to wait_seconds, by default 127 seconds, for
the Cassandra process to die. After this wait, it will throw an
exception stating it couldn't stop the node.
- wait_other_notice: return only when the other live nodes of the
cluster have marked this node has dead.
- gently: Let Scylla and Scylla JMX clean up and shut down properly.
Otherwise do a 'kill -9' which shuts down faster.
"""
marks = []
if self.is_running():
if wait_other_notice:
if not other_nodes:
other_nodes = list(self.cluster.nodes.values())
marks = [(node, node.mark_log()) for node in other_nodes
if node.is_live() and node is not self]
self._update_jmx_pid()
if self._process_jmx and self._process_scylla:
if gently:
try:
self._process_jmx.terminate()
except OSError as e:
pass
try:
self._process_scylla.terminate()
except OSError as e:
pass
else:
try:
self._process_jmx.kill()
except OSError as e:
pass
try:
self._process_scylla.kill()
except OSError as e:
pass
else:
signal_mapping = {True: signal.SIGTERM, False: signal.SIGKILL}
for pid in [self.jmx_pid, self.pid]:
try:
os.kill(pid, signal_mapping[gently])
except OSError:
pass
if wait_other_notice:
for node, mark in marks:
node.watch_log_for_death(self, from_mark=mark)
else:
time.sleep(.1)
still_running = self.is_running()
if still_running and wait:
# The sum of 7 sleeps starting at 1 and doubling each time
# is 2**7-1 (=127). So to sleep an arbitrary wait_seconds
# we need the first sleep to be wait_seconds/(2**7-1).
wait_time_sec = wait_seconds / (2**7 - 1.0)
for i in xrange(0, 7):
time.sleep(wait_time_sec)
if not self.is_running():
return True
wait_time_sec *= 2
raise NodeError("Problem stopping node %s" % self.name)
else:
return True
else:
return False
def start(self,
join_ring=True,
no_wait=False,
verbose=False,
update_pid=True,
wait_other_notice=False,
replace_token=None,
replace_address=None,
jvm_args=None,
wait_for_binary_proto=False,
profile_options=None,
use_jna=False,
quiet_start=False):
"""
Start the node. Options includes:
- join_ring: if false, start the node with -Dcassandra.join_ring=False
- no_wait: by default, this method returns when the node is started
and listening to clients.
If no_wait=True, the method returns sooner.
- wait_other_notice: if True, this method returns only when all other
live node of the cluster
have marked this node UP.
- replace_token: start the node with the -Dcassandra.replace_token
option.
- replace_address: start the node with the
-Dcassandra.replace_address option.
"""
if jvm_args is None:
jvm_args = []
scylla_cassandra_mapping = {
'-Dcassandra.replace_address_first_boot':
'--replace-address-first-boot'
}
# Replace args in the form
# ['-Dcassandra.foo=bar'] to ['-Dcassandra.foo', 'bar']
translated_args = []
new_jvm_args = []
for jvm_arg in jvm_args:
if '=' in jvm_arg:
split_option = jvm_arg.split("=")
e_msg = ("Option %s not in the form '-Dcassandra.foo=bar'. "
"Please check your test" % jvm_arg)
assert len(split_option) == 2, e_msg
option, value = split_option
# If we have information on how to translate the jvm option,
# translate it
if option in scylla_cassandra_mapping:
translated_args += [
scylla_cassandra_mapping[option], value
]
# Otherwise, just pass it as is
else:
new_jvm_args.append(jvm_arg)
else:
new_jvm_args.append(jvm_arg)
jvm_args = new_jvm_args
if self.is_running():
raise NodeError("%s is already running" % self.name)
for itf in list(self.network_interfaces.values()):
if itf is not None and replace_address is None:
try:
common.check_socket_available(itf)
except Exception as msg:
print("{}. Looking for offending processes...".format(msg))
for proc in psutil.process_iter():
if any(self.cluster.ipprefix in cmd
for cmd in proc.cmdline()):
print("name={} pid={} cmdline={}".format(
proc.name(), proc.pid, proc.cmdline()))
raise msg
marks = []
if wait_other_notice:
marks = [(node, node.mark_log())
for node in list(self.cluster.nodes.values())
if node.is_running()]
self.mark = self.mark_log()
launch_bin = common.join_bin(self.get_path(), 'bin', 'scylla')
options_file = os.path.join(self.get_path(), 'conf', 'scylla.yaml')
os.chmod(launch_bin, os.stat(launch_bin).st_mode | stat.S_IEXEC)
# TODO: we do not support forcing specific settings
# TODO: workaround for api-address as we do not load it
# from config file scylla#59
conf_file = os.path.join(self.get_conf_dir(), common.SCYLLA_CONF)
with open(conf_file, 'r') as f:
data = yaml.safe_load(f)
jvm_args = jvm_args + ['--api-address', data['api_address']]
jvm_args = jvm_args + [
'--collectd-hostname',
'%s.%s' % (socket.gethostname(), self.name)
]
# Let's add jvm_args and the translated args
args = [
launch_bin, '--options-file', options_file, '--log-to-stdout', '1'
] + jvm_args + translated_args
# Lets search for default overrides in SCYLLA_EXT_OPTS
scylla_ext_opts = os.getenv('SCYLLA_EXT_OPTS', "").split()
opts_i = 0
orig_args = list(args)
while opts_i < len(scylla_ext_opts):
if scylla_ext_opts[opts_i].startswith("--scylla-manager="):
opts_i += 1
elif scylla_ext_opts[opts_i].startswith('-'):
add = False
if scylla_ext_opts[opts_i] not in orig_args:
add = True
args.append(scylla_ext_opts[opts_i])
opts_i += 1
while opts_i < len(scylla_ext_opts) and not scylla_ext_opts[
opts_i].startswith('-'):
if add:
args.append(scylla_ext_opts[opts_i])
opts_i += 1
if '--developer-mode' not in args:
args += ['--developer-mode', 'true']
if '--smp' not in args:
# If --smp is not passed from cmdline, use default (--smp 1)
args += ['--smp', str(self._smp)]
elif self._smp_set_during_test:
# If node.set_smp() is called during the test, ignore the --smp
# passed from the cmdline.
args[args.index('--smp') + 1] = str(self._smp)
else:
# Update self._smp based on command line parameter.
# It may be used below, along with self._mem_mb_per_cpu, for calculating --memory
self._smp = int(args[args.index('--smp') + 1])
if '--memory' not in args:
# If --memory is not passed from cmdline, use default (512M per cpu)
args += [
'--memory', '{}M'.format(self._mem_mb_per_cpu * self._smp)
]
elif self._mem_set_during_test:
# If node.set_mem_mb_per_cpu() is called during the test, ignore the --memory
# passed from the cmdline.
args[args.index('--memory') + 1] = '{}M'.format(
self._mem_mb_per_cpu * self._smp)
if '--default-log-level' not in args:
args += ['--default-log-level', self.__global_log_level]
# TODO add support for classes_log_level
if '--collectd' not in args:
args += ['--collectd', '0']
if '--cpuset' not in args:
args += ['--overprovisioned']
if '--prometheus-address' not in args:
args += ['--prometheus-address', data['api_address']]
if replace_address:
args += ['--replace-address', replace_address]
args += ['--unsafe-bypass-fsync', '1']
scylla_process = self._start_scylla(args, marks, update_pid,
wait_other_notice,
wait_for_binary_proto)
self._start_jmx(data)
if not self._wait_java_up(data):
e_msg = (
"Error starting node %s: unable to connect to scylla-jmx" %
self.name)
raise NodeError(e_msg, scylla_process)
self.is_running()
return scylla_process
def stop(self, wait=True, wait_other_notice=False, gently=True):
"""
Stop the node.
- wait: if True (the default), wait for the Scylla process to be
really dead. Otherwise return after having sent the kill signal.
- wait_other_notice: return only when the other live nodes of the
cluster have marked this node has dead.
- gently: Let Scylla and Scylla JMX clean up and shut down properly.
Otherwise do a 'kill -9' which shuts down faster.
"""
marks = []
if self.is_running():
if wait_other_notice:
marks = [(node, node.mark_log())
for node in list(self.cluster.nodes.values())
if node.is_live() and node is not self]
self._update_jmx_pid()
if self._process_jmx and self._process_scylla:
if gently:
try:
self._process_jmx.terminate()
except OSError as e:
pass
try:
self._process_scylla.terminate()
except OSError as e:
pass
else:
try:
self._process_jmx.kill()
except OSError as e:
pass
try:
self._process_scylla.kill()
except OSError as e:
pass
else:
signal_mapping = {True: signal.SIGTERM, False: signal.SIGKILL}
for pid in [self.jmx_pid, self.pid]:
try:
os.kill(pid, signal_mapping[gently])
except OSError:
pass
if wait_other_notice:
for node, mark in marks:
node.watch_log_for_death(self, from_mark=mark)
else:
time.sleep(.1)
still_running = self.is_running()
if still_running and wait:
wait_time_sec = 1
for i in xrange(0, 7):
time.sleep(wait_time_sec)
if not self.is_running():
return True
wait_time_sec *= 2
raise NodeError("Problem stopping node %s" % self.name)
else:
return True
else:
return False
def start(self,
no_wait=False,
verbose=False,
wait_for_binary_proto=False,
wait_other_notice=False,
jvm_args=None,
profile_options=None,
quiet_start=False):
if not self.started and self.force_wait_for_cluster_start:
wait_other_notice = True
wait_for_binary_proto = True
self.started = True
p = None
if jvm_args is None:
jvm_args = []
marks = []
if wait_other_notice:
marks = [(node, node.mark_log()) for node in self.nodes.values()]
started = []
for node in self.nodes.values():
if not node.is_running():
mark = 0
if os.path.exists(node.logfilename()):
mark = node.mark_log()
p = node.start(update_pid=False,
jvm_args=jvm_args,
profile_options=profile_options)
# Let's ensure the nodes start at different times to avoid
# race conditions while creating system tables
time.sleep(1)
started.append((node, p, mark))
if no_wait and not verbose:
# waiting 2 seconds to check for early errors and for the
# pid to be set
time.sleep(2)
else:
for node, p, mark in started:
start_message = "Starting listening for CQL clients"
try:
# updated code, scylla starts CQL only by default
# process should not be checked for scylla as the
# process is a boot script (that ends after boot)
node.watch_log_for(start_message,
timeout=600,
verbose=verbose,
from_mark=mark)
except RuntimeError:
raise Exception("Not able to find start "
"message '%s' in Node '%s'" %
(start_message, node.name))
self.__update_pids(started)
for node, p, _ in started:
if not node.is_running():
raise NodeError("Error starting {0}.".format(node.name), p)
if not no_wait and self.cassandra_version() >= "0.8":
# 0.7 gossip messages seems less predictable that from 0.8
# onwards and I don't care enough
for node, _, mark in started:
for other_node, _, _ in started:
if other_node is not node:
node.watch_log_for_alive(other_node, from_mark=mark)
if wait_other_notice:
for old_node, mark in marks:
for node, _, _ in started:
if old_node is not node:
old_node.watch_log_for_alive(node, from_mark=mark)
if wait_for_binary_proto and self.version() >= '1.2':
for node, _, mark in started:
node.watch_log_for("Starting listening for CQL clients",
verbose=verbose,
from_mark=mark)
time.sleep(0.2)
if self._scylla_manager:
self._scylla_manager.start()
return started
