def _update_log4j(self):
super(DseNode, self)._update_log4j()
conf_file = os.path.join(self.get_conf_dir(), common.LOG4J_CONF)
append_pattern = 'log4j.appender.V.File='
log_file = os.path.join(self.get_path(), 'logs', 'solrvalidation.log')
if common.is_win():
log_file = re.sub("\\\\", "/", log_file)
common.replace_in_file(conf_file, append_pattern,
append_pattern + log_file)
append_pattern = 'log4j.appender.A.File='
log_file = os.path.join(self.get_path(), 'logs', 'audit.log')
if common.is_win():
log_file = re.sub("\\\\", "/", log_file)
common.replace_in_file(conf_file, append_pattern,
append_pattern + log_file)
append_pattern = 'log4j.appender.B.File='
log_file = os.path.join(self.get_path(), 'logs', 'audit',
'dropped-events.log')
if common.is_win():
log_file = re.sub("\\\\", "/", log_file)
common.replace_in_file(conf_file, append_pattern,
append_pattern + log_file)
def __init__(self,
path,
name,
partitioner=None,
install_dir=None,
create_directory=True,
version=None,
verbose=False,
**kwargs):
self.name = name
self.nodes = {}
self.seeds = []
self.partitioner = partitioner
self._config_options = {}
self._dse_config_options = {}
self.__log_level = "INFO"
self.__path = path
self.__version = None
self.use_vnodes = False
# Classes that are to follow the respective logging level
self._debug = []
self._trace = []
self.data_dir_count = 1
if self.name.lower() == "current":
raise RuntimeError("Cannot name a cluster 'current'.")
# This is incredibly important for
# backwards compatibility.
if 'cassandra_version' in kwargs:
version = kwargs['cassandra_version']
if 'cassandra_dir' in kwargs:
install_dir = kwargs['cassandra_dir']
if create_directory:
# we create the dir before potentially downloading to throw an error sooner if need be
os.mkdir(self.get_path())
try:
if version is None:
# at this point, install_dir should always not be None, but
# we keep this for backward compatibility (in loading old cluster)
if install_dir is not None:
if common.is_win():
self.__install_dir = install_dir
else:
self.__install_dir = os.path.abspath(install_dir)
self.__version = self.__get_version_from_build()
else:
dir, v = self.load_from_repository(version, verbose)
self.__install_dir = dir
self.__version = v if v is not None else self.__get_version_from_build(
)
if create_directory:
common.validate_install_dir(self.__install_dir)
self._update_config()
except:
if create_directory:
common.rmdirs(self.get_path())
raise
def __init__(self, path, name, partitioner=None, cassandra_dir=None, create_directory=True, cassandra_version=None, verbose=False):
self.name = name
self.nodes = {}
self.seeds = []
self.partitioner = partitioner
self._config_options = {}
self.__log_level = "INFO"
self.__path = path
self.__version = None
if create_directory:
# we create the dir before potentially downloading to throw an error sooner if need be
os.mkdir(self.get_path())
try:
if cassandra_version is None:
# at this point, cassandra_dir should always not be None, but
# we keep this for backward compatibility (in loading old cluster)
if cassandra_dir is not None:
if common.is_win():
self.__cassandra_dir = cassandra_dir
else:
self.__cassandra_dir = os.path.abspath(cassandra_dir)
self.__version = self.__get_version_from_build()
else:
dir, v = repository.setup(cassandra_version, verbose)
self.__cassandra_dir = dir
self.__version = v if v is not None else self.__get_version_from_build()
if create_directory:
common.validate_cassandra_dir(self.__cassandra_dir)
self.__update_config()
except:
if create_directory:
shutil.rmtree(self.get_path())
raise
def upgrade_to_version(self, tag, node):
format_args = {'node': node.name, 'tag': tag}
debug('Upgrading node {node} to {tag}'.format(**format_args))
# drain and shutdown
node.drain()
node.watch_log_for("DRAINED")
node.stop(wait_other_notice=False)
debug('{node} stopped'.format(**format_args))
# Ignore errors before upgrade on Windows
if is_win():
node.mark_log_for_errors()
# Update Cassandra Directory
debug('Updating version to tag {tag}'.format(**format_args))
node.set_install_dir(version=tag, verbose=True)
debug('Set new cassandra dir for {node}: {tag}'.format(**format_args))
# Restart node on new version
debug('Starting {node} on new version ({tag})'.format(**format_args))
# Setup log4j / logback again (necessary moving from 2.0 -> 2.1):
node.set_log_level("INFO")
node.start(wait_other_notice=True)
# wait for the conversion of legacy data to either complete or fail
# (because not enough upgraded nodes are available yet)
debug('Waiting for conversion of legacy data to complete or fail')
node.watch_log_for('conversion of legacy permissions')
debug('Running upgradesstables')
node.nodetool('upgradesstables -a')
debug('Upgrade of {node} complete'.format(**format_args))
def patient_cql_connection(self, node, keyspace=None,
user=None, password=None, timeout=30, compression=True,
protocol_version=None, port=None, ssl_opts=None, **kwargs):
"""
Returns a connection after it stops throwing NoHostAvailables due to not being ready.
If the timeout is exceeded, the exception is raised.
"""
if is_win():
timeout *= 2
expected_log_lines = ('Control connection failed to connect, shutting down Cluster:',
'[control connection] Error connecting to ')
with log_filter('cassandra.cluster', expected_log_lines):
session = retry_till_success(
self.cql_connection,
node,
keyspace=keyspace,
user=user,
password=password,
timeout=timeout,
compression=compression,
protocol_version=protocol_version,
port=port,
ssl_opts=ssl_opts,
bypassed_exception=NoHostAvailable,
**kwargs
)
return session
def watch_log_for(self, exprs, from_mark=None, timeout=600, process=None, verbose=False):
"""
Watch the log until one or more (regular) expression are found.
This methods when all the expressions have been found or the method
timeouts (a TimeoutError is then raised). On successful completion,
a list of pair (line matched, match object) is returned.
"""
elapsed = 0
tofind = [exprs] if isinstance(exprs, string_types) else exprs
tofind = [ re.compile(e) for e in tofind ]
matchings = []
reads = ""
if len(tofind) == 0:
return None
while not os.path.exists(self.logfilename()):
time.sleep(.5)
if process:
process.poll()
if process.returncode is not None:
self.print_process_output(self.name, process, verbose)
if process.returncode != 0:
raise RuntimeError() # Shouldn't reuse RuntimeError but I'm lazy
with open(self.logfilename()) as f:
if from_mark:
f.seek(from_mark)
while True:
# First, if we have a process to check, then check it.
# Skip on Windows - stdout/stderr is cassandra.bat
if not common.is_win():
if process:
process.poll()
if process.returncode is not None:
self.print_process_output(self.name, process, verbose)
if process.returncode != 0:
raise RuntimeError() # Shouldn't reuse RuntimeError but I'm lazy
line = f.readline()
if line:
reads = reads + line
for e in tofind:
m = e.search(line)
if m:
matchings.append((line, m))
tofind.remove(e)
if len(tofind) == 0:
return matchings[0] if isinstance(exprs, string_types) else matchings
else:
# yep, it's ugly
time.sleep(1)
elapsed = elapsed + 1
if elapsed > timeout:
raise TimeoutError(time.strftime("%d %b %Y %H:%M:%S", time.gmtime()) + " [" + self.name + "] Missing: " + str([e.pattern for e in tofind]) + ":\n" + reads)
if process:
process.poll()
if process.returncode is not None and process.returncode == 0:
return None
def copy_logs(self, directory=None, name=None):
"""Copy the current cluster's log files somewhere, by default to LOG_SAVED_DIR with a name of 'last'"""
if directory is None:
directory = self.log_saved_dir
if name is None:
name = self.last_log
else:
name = os.path.join(directory, name)
if not os.path.exists(directory):
os.mkdir(directory)
logs = [(node.name, node.logfilename(), node.debuglogfilename(), node.gclogfilename(),
node.compactionlogfilename())
for node in self.cluster.nodelist()]
if len(logs) is not 0:
basedir = str(int(time.time() * 1000)) + '_' + str(id(self))
logdir = os.path.join(directory, basedir)
os.mkdir(logdir)
for n, log, debuglog, gclog, compactionlog in logs:
if os.path.exists(log):
assert os.path.getsize(log) >= 0
shutil.copyfile(log, os.path.join(logdir, n + ".log"))
if os.path.exists(debuglog):
assert os.path.getsize(debuglog) >= 0
shutil.copyfile(debuglog, os.path.join(logdir, n + "_debug.log"))
if os.path.exists(gclog):
assert os.path.getsize(gclog) >= 0
shutil.copyfile(gclog, os.path.join(logdir, n + "_gc.log"))
if os.path.exists(compactionlog):
assert os.path.getsize(compactionlog) >= 0
shutil.copyfile(compactionlog, os.path.join(logdir, n + "_compaction.log"))
if os.path.exists(name):
os.unlink(name)
if not is_win():
os.symlink(basedir, name)
def validate(self, parser, options, args):
Cmd.validate(self, parser, options, args, cluster_name=True)
if options.ipprefix and options.ipformat:
parser.print_help()
parser.error("%s and %s may not be used together" % (parser.get_option('-i'), parser.get_option('-I')))
self.nodes = parse_populate_count(options.nodes)
if self.options.vnodes and self.nodes is None:
print_("Can't set --vnodes if not populating cluster in this command.")
parser.print_help()
exit(1)
if not options.version:
try:
common.validate_install_dir(options.install_dir)
except ArgumentError:
parser.print_help()
parser.error("%s is not a valid cassandra directory. You must define a cassandra dir or version." % options.install_dir)
if common.get_dse_version(options.install_dir) is not None:
common.assert_jdk_valid_for_cassandra_version(common.get_dse_cassandra_version(options.install_dir))
else:
common.assert_jdk_valid_for_cassandra_version(common.get_version_from_build(options.install_dir))
if common.is_win() and os.path.exists('c:\windows\system32\java.exe'):
print_("""WARN: c:\windows\system32\java.exe exists.
This may cause registry issues, and jre7 to be used, despite jdk8 being installed.
""")
def patient_exclusive_cql_connection(self, node, keyspace=None,
user=None, password=None, timeout=30, compression=True,
protocol_version=None, port=None, ssl_opts=None, **kwargs):
"""
Returns a connection after it stops throwing NoHostAvailables due to not being ready.
If the timeout is exceeded, the exception is raised.
"""
if is_win():
timeout *= 2
return retry_till_success(
self.exclusive_cql_connection,
node,
keyspace=keyspace,
user=user,
password=password,
timeout=timeout,
compression=compression,
protocol_version=protocol_version,
port=port,
ssl_opts=ssl_opts,
bypassed_exception=NoHostAvailable,
**kwargs
)
def __init__(self, path, name, partitioner=None, install_dir=None, create_directory=True, version=None, verbose=False, snitch='org.apache.cassandra.locator.PropertyFileSnitch', **kwargs):
self.name = name
self.id = 0
self.ipprefix = None
self.ipformat = None
self.nodes = {}
self.seeds = []
self.partitioner = partitioner
self.snitch = snitch
self._config_options = {}
self._dse_config_options = {}
self.__log_level = "INFO"
self.path = path
self.__version = None
self.use_vnodes = False
# Classes that are to follow the respective logging level
self._debug = []
self._trace = []
if self.name.lower() == "current":
raise RuntimeError("Cannot name a cluster 'current'.")
# This is incredibly important for
# backwards compatibility.
version = kwargs.get('cassandra_version', version)
install_dir = kwargs.get('cassandra_dir', install_dir)
docker_image = kwargs.get('docker_image')
if create_directory:
# we create the dir before potentially downloading to throw an error sooner if need be
os.mkdir(self.get_path())
if docker_image:
self.docker_image = docker_image
self.__install_dir = None
self.__version = '3.0' # TODO: add option to read the version from docker image
return
try:
if version is None:
# at this point, install_dir should always not be None, but
# we keep this for backward compatibility (in loading old cluster)
if install_dir is not None:
if common.is_win():
self.__install_dir = install_dir
else:
self.__install_dir = os.path.abspath(install_dir)
self.__version = self.__get_version_from_build()
else:
dir, v = self.load_from_repository(version, verbose)
self.__install_dir = dir
self.__version = v if v is not None else self.__get_version_from_build()
if create_directory:
common.validate_install_dir(self.__install_dir)
self._update_config()
except:
if create_directory:
common.rmdirs(self.get_path())
raise
self.debug("Started cluster '{}' version {} installed in {}".format(self.name, self.__version, self.__install_dir))
def __init__(
self,
path,
name,
partitioner=None,
install_dir=None,
create_directory=True,
version=None,
verbose=False,
**kwargs
):
self.name = name
self.nodes = {}
self.seeds = []
self.partitioner = partitioner
self._config_options = {}
self._dse_config_options = {}
self.__log_level = "INFO"
self.__path = path
self.__version = None
self.use_vnodes = False
# Classes that are to follow the respective logging level
self._debug = []
self._trace = []
if self.name.lower() == "current":
raise RuntimeError("Cannot name a cluster 'current'.")
##This is incredibly important for
##backwards compatibility.
if "cassandra_version" in kwargs:
version = kwargs["cassandra_version"]
if "cassandra_dir" in kwargs:
install_dir = kwargs["cassandra_dir"]
if create_directory:
# we create the dir before potentially downloading to throw an error sooner if need be
os.mkdir(self.get_path())
try:
if version is None:
# at this point, install_dir should always not be None, but
# we keep this for backward compatibility (in loading old cluster)
if install_dir is not None:
if common.is_win():
self.__install_dir = install_dir
else:
self.__install_dir = os.path.abspath(install_dir)
self.__version = self.__get_version_from_build()
else:
dir, v = self.load_from_repository(version, verbose)
self.__install_dir = dir
self.__version = v if v is not None else self.__get_version_from_build()
if create_directory:
common.validate_install_dir(self.__install_dir)
self._update_config()
except:
if create_directory:
common.rmdirs(self.get_path())
raise
def validate(self, parser, options, args):
Cmd.validate(self, parser, options, args, cluster_name=True)
if options.ipprefix and options.ipformat:
parser.print_help()
parser.error("%s and %s may not be used together" % (parser.get_option('-i'), parser.get_option('-I')))
self.nodes = parse_populate_count(options.nodes)
if self.options.vnodes and self.nodes is None:
print_("Can't set --vnodes if not populating cluster in this command.")
parser.print_help()
exit(1)
if not options.version:
try:
common.validate_install_dir(options.install_dir)
except ArgumentError:
parser.print_help()
parser.error("%s is not a valid cassandra directory. You must define a cassandra dir or version." % options.install_dir)
if common.get_dse_version(options.install_dir) is not None:
common.assert_jdk_valid_for_cassandra_version(common.get_dse_cassandra_version(options.install_dir))
else:
common.assert_jdk_valid_for_cassandra_version(common.get_version_from_build(options.install_dir))
if common.is_win() and os.path.exists('c:\windows\system32\java.exe'):
print_("""WARN: c:\windows\system32\java.exe exists.
This may cause registry issues, and jre7 to be used, despite jdk8 being installed.
""")
def validate(self, parser, options, args):
if options.scylla and not options.install_dir:
parser.error("must specify install_dir using scylla")
Cmd.validate(self, parser, options, args, cluster_name=True)
if options.ipprefix and options.ipformat:
parser.print_help()
parser.error("%s and %s may not be used together" % (parser.get_option('-i'), parser.get_option('-I')))
self.nodes = parse_populate_count(options.nodes)
if self.options.vnodes and self.nodes is None:
print_("Can't set --vnodes if not populating cluster in this command.")
parser.print_help()
sys.exit(1)
if self.options.snitch and \
(not isinstance(self.nodes, list) or
not (self.options.snitch == 'org.apache.cassandra.locator.PropertyFileSnitch' or
self.options.snitch == 'org.apache.cassandra.locator.GossipingPropertyFileSnitch')):
parser.print_help()
sys.exit(1)
if not options.version:
try:
common.validate_install_dir(options.install_dir)
except ArgumentError:
parser.print_help()
parser.error("%s is not a valid cassandra directory. You must define a cassandra dir or version." % options.install_dir)
common.assert_jdk_valid_for_cassandra_version(common.get_version_from_build(options.install_dir))
if common.is_win() and os.path.exists('c:\windows\system32\java.exe'):
print_("""WARN: c:\windows\system32\java.exe exists.
This may cause registry issues, and jre7 to be used, despite jdk8 being installed.
""")
def __update_status(self):
if self.pid is None:
if self.status == Status.UP or self.status == Status.DECOMMISIONNED:
self.status = Status.DOWN
return
old_status = self.status
# os.kill on windows doesn't allow us to ping a process
if common.is_win():
self.__update_status_win()
else:
try:
os.kill(self.pid, 0)
except OSError as err:
if err.errno == errno.ESRCH:
# not running
if self.status == Status.UP or self.status == Status.DECOMMISIONNED:
self.status = Status.DOWN
elif err.errno == errno.EPERM:
# no permission to signal this process
if self.status == Status.UP or self.status == Status.DECOMMISIONNED:
self.status = Status.DOWN
else:
# some other error
raise err
else:
if self.status == Status.DOWN or self.status == Status.UNINITIALIZED:
self.status = Status.UP
if not old_status == self.status:
if old_status == Status.UP and self.status == Status.DOWN:
self.pid = None
self.__update_config()
def copy_logs(self, directory=None, name=None):
"""Copy the current cluster's log files somewhere, by default to LOG_SAVED_DIR with a name of 'last'"""
if directory is None:
directory = LOG_SAVED_DIR
if name is None:
name = LAST_LOG
else:
name = os.path.join(directory, name)
if not os.path.exists(directory):
os.mkdir(directory)
logs = [(node.name, node.logfilename(), node.debuglogfilename())
for node in self.cluster.nodes.values()]
if len(logs) is not 0:
basedir = str(int(time.time() * 1000)) + '_' + self.id()
logdir = os.path.join(directory, basedir)
os.mkdir(logdir)
for n, log, debuglog in logs:
if os.path.exists(log):
shutil.copyfile(log, os.path.join(logdir, n + ".log"))
if os.path.exists(debuglog):
shutil.copyfile(debuglog,
os.path.join(logdir, n + "_debug.log"))
if os.path.exists(name):
os.unlink(name)
if not is_win():
os.symlink(basedir, name)
def patient_exclusive_cql_connection(self,
node,
keyspace=None,
user=None,
password=None,
timeout=30,
compression=True,
protocol_version=None,
port=None,
ssl_opts=None,
**kwargs):
"""
Returns a connection after it stops throwing NoHostAvailables due to not being ready.
If the timeout is exceeded, the exception is raised.
"""
if is_win():
timeout *= 2
return retry_till_success(self.exclusive_cql_connection,
node,
keyspace=keyspace,
user=user,
password=password,
timeout=timeout,
compression=compression,
protocol_version=protocol_version,
port=port,
ssl_opts=ssl_opts,
bypassed_exception=NoHostAvailable,
**kwargs)
def stress(self, stress_options):
stress = common.get_stress_bin(self.get_install_dir())
livenodes = [
node.network_interfaces['storage'][0]
for node in list(self.nodes.values()) if node.is_live()
]
if len(livenodes) == 0:
print_("No live node")
return
nodes_options = []
if self.cassandra_version() <= '2.1':
if '-d' not in stress_options:
nodes_options = ['-d', ",".join(livenodes)]
args = [stress] + nodes_options + stress_options
else:
if '-node' not in stress_options:
nodes_options = ['-node', ','.join(livenodes)]
args = [stress] + stress_options + nodes_options
rc = None
try:
# need to set working directory for env on Windows
if common.is_win():
rc = subprocess.call(args, cwd=common.parse_path(stress))
else:
rc = subprocess.call(args)
except KeyboardInterrupt:
pass
return rc
def get_parser(self):
usage = "usage: ccm node stop [options] name"
parser = self._get_default_parser(usage, self.description())
parser.add_option('--no-wait',
action="store_true",
dest="no_wait",
help="Do not wait for the node to be stopped",
default=False)
parser.add_option('-g',
'--gently',
action="store_const",
dest="signal_event",
help="Shut down gently (default)",
const=signal.SIGTERM,
default=signal.SIGTERM)
if common.is_win():
# Fill the dictionary with SIGTERM as the cluster is killed forcefully
# on Windows regardless of assigned signal (TASKKILL is used)
default_signal_events = {'1': signal.SIGTERM, '9': signal.SIGTERM}
else:
default_signal_events = {'1': signal.SIGHUP, '9': signal.SIGKILL}
parser.add_option('--hang-up',
action="store_const",
dest="signal_event",
help="Shut down via hang up (kill -1)",
const=default_signal_events['1'])
parser.add_option('--not-gently',
action="store_const",
dest="signal_event",
help="Shut down immediately (kill -9)",
const=default_signal_events['9'])
return parser
def apply_jmx_authentication(node):
replacement_list = [
(r'#\$env:JVM_OPTS="\$env:JVM_OPTS -Dcom.sun.management.jmxremote.authenticate=true"',
'$env:JVM_OPTS="$env:JVM_OPTS -Dcom.sun.management.jmxremote.authenticate=true"'
),
(r'#\$env:JVM_OPTS="\$env:JVM_OPTS -Dcassandra.jmx.remote.login.config=CassandraLogin"',
'$env:JVM_OPTS="$env:JVM_OPTS -Dcassandra.jmx.remote.login.config=CassandraLogin"'
),
(r'#\$env:JVM_OPTS="\$env:JVM_OPTS -Djava.security.auth.login.config=C:/cassandra-jaas.config"',
r'$env:JVM_OPTS="$env:JVM_OPTS -Djava.security.auth.login.config=$env:CASSANDRA_CONF\cassandra-jaas.config"'
),
(r'#\$env:JVM_OPTS="\$env:JVM_OPTS -Dcassandra.jmx.authorizer=org.apache.cassandra.auth.jmx.AuthorizationProxy"',
'$env:JVM_OPTS="$env:JVM_OPTS -Dcassandra.jmx.authorizer=org.apache.cassandra.auth.jmx.AuthorizationProxy"'
)
] if common.is_win() else [
(r'JVM_OPTS="\$JVM_OPTS -Dcom.sun.management.jmxremote.authenticate=false"',
'JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.authenticate=true"'
),
(r'JVM_OPTS="\$JVM_OPTS -Dcom.sun.management.jmxremote.password.file=/etc/cassandra/jmxremote.password"',
'#JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.password.file=/etc/cassandra/jmxremote.password"'
),
(r'#JVM_OPTS="\$JVM_OPTS -Dcassandra.jmx.remote.login.config=CassandraLogin"',
'JVM_OPTS="$JVM_OPTS -Dcassandra.jmx.remote.login.config=CassandraLogin"'
),
(r'#JVM_OPTS="\$JVM_OPTS -Djava.security.auth.login.config=\$CASSANDRA_CONF/cassandra-jaas.config"',
'JVM_OPTS="$JVM_OPTS -Djava.security.auth.login.config=$CASSANDRA_CONF/cassandra-jaas.config"'
),
(r'#JVM_OPTS="\$JVM_OPTS -Dcassandra.jmx.authorizer=org.apache.cassandra.auth.jmx.AuthorizationProxy"',
'JVM_OPTS="$JVM_OPTS -Dcassandra.jmx.authorizer=org.apache.cassandra.auth.jmx.AuthorizationProxy"'
)
]
common.replaces_in_file(node.envfilename(), replacement_list)
def stress(self, stress_options):
stress = common.get_stress_bin(self.get_install_dir())
livenodes = [node.network_interfaces['storage'][0] for node in list(self.nodes.values()) if node.is_live()]
if len(livenodes) == 0:
print_("No live node")
return
nodes_options = []
if self.cassandra_version() <= '2.1':
if '-d' not in stress_options:
nodes_options = ['-d', ",".join(livenodes)]
args = [stress] + nodes_options + stress_options
else:
if '-node' not in stress_options:
nodes_options = ['-node', ','.join(livenodes)]
args = [stress] + stress_options + nodes_options
rc = None
try:
# need to set working directory for env on Windows
if common.is_win():
rc = subprocess.call(args, cwd=common.parse_path(stress))
else:
rc = subprocess.call(args)
except KeyboardInterrupt:
pass
return rc
def watch_log_for(self, exprs, from_mark=None, timeout=600, process=None, verbose=False):
"""
Watch the log until one or more (regular) expression are found.
This methods when all the expressions have been found or the method
timeouts (a TimeoutError is then raised). On successful completion,
a list of pair (line matched, match object) is returned.
"""
elapsed = 0
tofind = [exprs] if isinstance(exprs, string_types) else exprs
tofind = [ re.compile(e) for e in tofind ]
matchings = []
reads = ""
if len(tofind) == 0:
return None
while not os.path.exists(self.logfilename()):
time.sleep(.5)
if process:
process.poll()
if process.returncode is not None:
self.print_process_output(self.name, process, verbose)
if process.returncode != 0:
raise RuntimeError() # Shouldn't reuse RuntimeError but I'm lazy
with open(self.logfilename()) as f:
if from_mark:
f.seek(from_mark)
while True:
# First, if we have a process to check, then check it.
# Skip on Windows - stdout/stderr is cassandra.bat
if not common.is_win():
if process:
process.poll()
if process.returncode is not None:
self.print_process_output(self.name, process, verbose)
if process.returncode != 0:
raise RuntimeError() # Shouldn't reuse RuntimeError but I'm lazy
line = f.readline()
if line:
reads = reads + line
for e in tofind:
m = e.search(line)
if m:
matchings.append((line, m))
tofind.remove(e)
if len(tofind) == 0:
return matchings[0] if isinstance(exprs, string_types) else matchings
else:
# yep, it's ugly
time.sleep(1)
elapsed = elapsed + 1
if elapsed > timeout:
raise TimeoutError(time.strftime("%d %b %Y %H:%M:%S", time.gmtime()) + " [" + self.name + "] Missing: " + str([e.pattern for e in tofind]) + ":\n" + reads)
if process:
process.poll()
if process.returncode is not None and process.returncode == 0:
return None
def __update_log4j(self):
append_pattern = 'log4j.appender.R.File='
conf_file = os.path.join(self.get_conf_dir(), common.LOG4J_CONF)
log_file = os.path.join(self.get_path(), 'logs', 'system.log')
# log4j isn't partial to Windows \. I can't imagine why not.
if common.is_win():
log_file = re.sub("\\\\", "/", log_file)
common.replace_in_file(conf_file, append_pattern,
append_pattern + log_file)
# Setting the right log level
# Replace the global log level
if self.__global_log_level is not None:
append_pattern = 'log4j.rootLogger='
common.replace_in_file(
conf_file, append_pattern,
append_pattern + self.__global_log_level + ',stdout,R')
# Class specific log levels
for class_name in self.__classes_log_level:
logger_pattern = 'log4j.logger'
full_logger_pattern = logger_pattern + '.' + class_name + '='
common.replace_or_add_into_file_tail(
conf_file, full_logger_pattern,
full_logger_pattern + self.__classes_log_level[class_name])
def __update_status(self):
if self.pid is None:
if self.status == Status.UP or self.status == Status.DECOMMISIONNED:
self.status = Status.DOWN
return
old_status = self.status
# os.kill on windows doesn't allow us to ping a process
if common.is_win():
self.__update_status_win()
else:
try:
os.kill(self.pid, 0)
except OSError as err:
if err.errno == errno.ESRCH:
# not running
if self.status == Status.UP or self.status == Status.DECOMMISIONNED:
self.status = Status.DOWN
elif err.errno == errno.EPERM:
# no permission to signal this process
if self.status == Status.UP or self.status == Status.DECOMMISIONNED:
self.status = Status.DOWN
else:
# some other error
raise err
else:
if self.status == Status.DOWN or self.status == Status.UNINITIALIZED:
self.status = Status.UP
if not old_status == self.status:
if old_status == Status.UP and self.status == Status.DOWN:
self.pid = None
self.__update_config()
def patient_cql_connection(self, node, keyspace=None,
user=None, password=None, timeout=30, compression=True,
protocol_version=None, port=None, ssl_opts=None):
"""
Returns a connection after it stops throwing NoHostAvailables due to not being ready.
If the timeout is exceeded, the exception is raised.
"""
if is_win():
timeout *= 2
logging.getLogger('cassandra.cluster').addFilter(expect_control_connection_failures)
try:
session = retry_till_success(
self.cql_connection,
node,
keyspace=keyspace,
user=user,
password=password,
timeout=timeout,
compression=compression,
protocol_version=protocol_version,
port=port,
ssl_opts=ssl_opts,
bypassed_exception=NoHostAvailable
)
finally:
logging.getLogger('cassandra.cluster').removeFilter(expect_control_connection_failures)
return session
def tearDown(self):
if self.tempfile:
if is_win():
self.tempfile.close()
os.unlink(self.tempfile.name)
super(CqlshCopyTest, self).tearDown()
def patient_cql_connection(self,
node,
keyspace=None,
version=None,
user=None,
password=None,
timeout=10,
compression=True,
protocol_version=None):
"""
Returns a connection after it stops throwing NoHostAvailables due to not being ready.
If the timeout is exceeded, the exception is raised.
"""
if is_win():
timeout = timeout * 5
return retry_till_success(self.cql_connection,
node,
keyspace=keyspace,
version=version,
user=user,
password=password,
timeout=timeout,
compression=compression,
protocol_version=protocol_version,
bypassed_exception=NoHostAvailable)
def copy_logs(self, directory=None, name=None):
"""Copy the current cluster's log files somewhere, by default to LOG_SAVED_DIR with a name of 'last'"""
if directory is None:
directory = LOG_SAVED_DIR
if name is None:
name = LAST_LOG
else:
name = os.path.join(directory, name)
if not os.path.exists(directory):
os.mkdir(directory)
logs = [(node.name, node.logfilename(), node.debuglogfilename(), node.gclogfilename(), node.compactionlogfilename()) for node in self.cluster.nodes.values()]
if len(logs) is not 0:
basedir = str(int(time.time() * 1000)) + '_' + self.id()
logdir = os.path.join(directory, basedir)
os.mkdir(logdir)
for n, log, debuglog, gclog, compactionlog in logs:
if os.path.exists(log):
self.assertGreaterEqual(os.path.getsize(log), 0)
shutil.copyfile(log, os.path.join(logdir, n + ".log"))
if os.path.exists(debuglog):
self.assertGreaterEqual(os.path.getsize(debuglog), 0)
shutil.copyfile(debuglog, os.path.join(logdir, n + "_debug.log"))
if os.path.exists(gclog):
self.assertGreaterEqual(os.path.getsize(gclog), 0)
shutil.copyfile(gclog, os.path.join(logdir, n + "_gc.log"))
if os.path.exists(compactionlog):
self.assertGreaterEqual(os.path.getsize(compactionlog), 0)
shutil.copyfile(compactionlog, os.path.join(logdir, n + "_compaction.log"))
if os.path.exists(name):
os.unlink(name)
if not is_win():
os.symlink(basedir, name)
def __init__(self, path, name, partitioner=None, cassandra_dir=None, create_directory=True, cassandra_version=None, verbose=False):
self.name = name
self.nodes = {}
self.seeds = []
self.partitioner = partitioner
self._config_options = {}
self.__log_level = "INFO"
self.__path = path
self.__version = None
if create_directory:
# we create the dir before potentially downloading to throw an error sooner if need be
os.mkdir(self.get_path())
try:
if cassandra_version is None:
# at this point, cassandra_dir should always not be None, but
# we keep this for backward compatibility (in loading old cluster)
if cassandra_dir is not None:
if common.is_win():
self.__cassandra_dir = cassandra_dir
else:
self.__cassandra_dir = os.path.abspath(cassandra_dir)
self.__version = self.__get_version_from_build()
else:
dir, v = repository.setup(cassandra_version, verbose)
self.__cassandra_dir = dir
self.__version = v if v is not None else self.__get_version_from_build()
if create_directory:
common.validate_cassandra_dir(self.__cassandra_dir)
self.__update_config()
except:
if create_directory:
shutil.rmtree(self.get_path())
raise
def around_test(self):
self.validate_class_config()
logger.info(
"Upgrade test beginning, setting CASSANDRA_VERSION to {}, and jdk to {}. (Prior values will be restored after test)."
.format(self.UPGRADE_PATH.starting_version,
self.UPGRADE_PATH.starting_meta.java_version))
previous_java_home = os.environ['JAVA_HOME']
previous_cassandra_version = os.environ[
'CASSANDRA_VERSION'] if 'CASSANDRA_VERSION' in os.environ else None
switch_jdks(self.UPGRADE_PATH.starting_meta.java_version)
os.environ['CASSANDRA_VERSION'] = self.UPGRADE_PATH.starting_version
yield
os.environ['JAVA_HOME'] = previous_java_home
if previous_cassandra_version:
os.environ['CASSANDRA_VERSION'] = previous_cassandra_version
# Ignore errors before upgrade on Windows
# We ignore errors from 2.1, because windows 2.1
# support is only beta. There are frequent log errors,
# related to filesystem interactions that are a direct result
# of the lack of full functionality on 2.1 Windows, and we dont
# want these to pollute our results.
if is_win() and self.cluster.version() <= '2.2':
self.cluster.nodelist()[1].mark_log_for_errors()
def patient_cql_connection(self, node, keyspace=None,
user=None, password=None, timeout=30, compression=True,
protocol_version=None, port=None, ssl_opts=None):
"""
Returns a connection after it stops throwing NoHostAvailables due to not being ready.
If the timeout is exceeded, the exception is raised.
"""
if is_win():
timeout *= 2
expected_log_lines = ('Control connection failed to connect, shutting down Cluster:', '[control connection] Error connecting to ')
with log_filter('cassandra.cluster', expected_log_lines):
session = retry_till_success(
self.cql_connection,
node,
keyspace=keyspace,
user=user,
password=password,
timeout=timeout,
compression=compression,
protocol_version=protocol_version,
port=port,
ssl_opts=ssl_opts,
bypassed_exception=NoHostAvailable
)
return session
def do_upgrade(self, session):
"""
Upgrades the first node in the cluster and returns a list of
(is_upgraded, Session) tuples. If `is_upgraded` is true, the
Session is connected to the upgraded node.
"""
session.cluster.shutdown()
node1 = self.cluster.nodelist()[0]
node2 = self.cluster.nodelist()[1]
# stop the nodes
node1.drain()
node1.stop(gently=True)
# Ignore errors before upgrade on Windows
# We ignore errors from 2.1, because windows 2.1
# support is only beta. There are frequent log errors,
# related to filesystem interactions that are a direct result
# of the lack of full functionality on 2.1 Windows, and we dont
# want these to pollute our results.
if is_win() and self.cluster.version() <= '2.2':
node1.mark_log_for_errors()
debug('upgrading node1 to {}'.format(self.UPGRADE_PATH.upgrade_version))
switch_jdks(self.UPGRADE_PATH.upgrade_meta.java_version)
node1.set_install_dir(version=self.UPGRADE_PATH.upgrade_version)
# this is a bandaid; after refactoring, upgrades should account for protocol version
new_version_from_build = get_version_from_build(node1.get_install_dir())
if (new_version_from_build >= '3' and self.protocol_version is not None and self.protocol_version < 3):
self.skip('Protocol version {} incompatible '
'with Cassandra version {}'.format(self.protocol_version, new_version_from_build))
node1.set_log_level("DEBUG" if DEBUG else "INFO")
node1.set_configuration_options(values={'internode_compression': 'none'})
node1.start(wait_for_binary_proto=True, wait_other_notice=True)
sessions = []
session = self.patient_exclusive_cql_connection(node1, protocol_version=self.protocol_version)
session.set_keyspace('ks')
sessions.append((True, session))
# open a second session with the node on the old version
session = self.patient_exclusive_cql_connection(node2, protocol_version=self.protocol_version)
session.set_keyspace('ks')
sessions.append((False, session))
if self.CL:
for is_upgraded, session in sessions:
session.default_consistency_level = self.CL
# Let the nodes settle briefly before yielding connections in turn (on the upgraded and non-upgraded alike)
# CASSANDRA-11396 was the impetus for this change, wherein some apparent perf noise was preventing
# CL.ALL from being reached. The newly upgraded node needs to settle because it has just barely started, and each
# non-upgraded node needs a chance to settle as well, because the entire cluster (or isolated nodes) may have been doing resource intensive activities
# immediately before.
for s in sessions:
time.sleep(5)
yield s
def launch_nodetool_cmd(self, cmd):
"""
Launch a nodetool command and check the result is empty (no error)
"""
node1 = self.cluster.nodelist()[0]
response = node1.nodetool(cmd, capture_output=True)[0]
if not common.is_win(): # nodetool always prints out on windows
assert len(response) == 0, response # nodetool does not print anything unless there is an error
def launch_nodetool_cmd(self, cmd):
"""
Launch a nodetool command and check the result is empty (no error)
"""
node1 = self.cluster.nodelist()[0]
response = node1.nodetool(cmd, capture_output=True)[0]
if not common.is_win(): # nodetool always prints out on windows
assert_length_equal(response, 0) # nodetool does not print anything unless there is an error
def launch_nodetool_cmd(self, cmd):
"""
Launch a nodetool command and check the result is empty (no error)
"""
node1 = self.cluster.nodelist()[0]
response = node1.nodetool(cmd).stdout
if not common.is_win(): # nodetool always prints out on windows
assert_length_equal(response, 0) # nodetool does not print anything unless there is an error
def setUp(self):
global CURRENT_TEST
CURRENT_TEST = self.id() + self._testMethodName
# On Windows, forcefully terminate any leftover previously running cassandra processes. This is a temporary
# workaround until we can determine the cause of intermittent hung-open tests and file-handles.
if is_win():
try:
import psutil
for proc in psutil.process_iter():
try:
pinfo = proc.as_dict(attrs=['pid', 'name', 'cmdline'])
except psutil.NoSuchProcess:
pass
else:
if (pinfo['name'] == 'java.exe' and '-Dcassandra' in pinfo['cmdline']):
print 'Found running cassandra process with pid: ' + str(pinfo['pid']) + '. Killing.'
psutil.Process(pinfo['pid']).kill()
except ImportError:
debug("WARN: psutil not installed. Cannot detect and kill running cassandra processes - you may see cascading dtest failures.")
# cleaning up if a previous execution didn't trigger tearDown (which
# can happen if it is interrupted by KeyboardInterrupt)
# TODO: move that part to a generic fixture
if os.path.exists(LAST_TEST_DIR):
with open(LAST_TEST_DIR) as f:
self.test_path = f.readline().strip('\n')
name = f.readline()
try:
self.cluster = ClusterFactory.load(self.test_path, name)
# Avoid waiting too long for node to be marked down
if not self._preserve_cluster:
self._cleanup_cluster()
except IOError:
# after a restart, /tmp will be emptied so we'll get an IOError when loading the old cluster here
pass
self.cluster = self._get_cluster()
if ENABLE_ACTIVE_LOG_WATCHING:
if not self.allow_log_errors:
self.begin_active_log_watch()
if RECORD_COVERAGE:
self.__setup_jacoco()
try:
self.init_config()
except NotImplementedError:
debug("Custom init_config not found. Setting defaults.")
self.init_default_config()
with open(LAST_TEST_DIR, 'w') as f:
f.write(self.test_path + '\n')
f.write(self.cluster.name)
self.modify_log(self.cluster)
self.connections = []
self.runners = []
self.maxDiff = None
def do_upgrade(self, session):
"""
Upgrades the first node in the cluster and returns a list of
(is_upgraded, Session) tuples. If `is_upgraded` is true, the
Session is connected to the upgraded node.
"""
session.cluster.shutdown()
node1 = self.cluster.nodelist()[0]
node2 = self.cluster.nodelist()[1]
# stop the nodes
node1.drain()
node1.stop(gently=True)
# Ignore errors before upgrade on Windows
# We ignore errors from 2.1, because windows 2.1
# support is only beta. There are frequent log errors,
# related to filesystem interactions that are a direct result
# of the lack of full functionality on 2.1 Windows, and we dont
# want these to pollute our results.
if is_win() and self.cluster.version() <= '2.2':
node1.mark_log_for_errors()
debug('upgrading node1 to {}'.format(self.UPGRADE_PATH.upgrade_version))
node1.set_install_dir(version=self.UPGRADE_PATH.upgrade_version)
# this is a bandaid; after refactoring, upgrades should account for protocol version
new_version_from_build = get_version_from_build(node1.get_install_dir())
if (new_version_from_build >= '3' and self.protocol_version is not None and self.protocol_version < 3):
self.skip('Protocol version {} incompatible '
'with Cassandra version {}'.format(self.protocol_version, new_version_from_build))
node1.set_log_level("DEBUG" if DEBUG else "INFO")
node1.set_configuration_options(values={'internode_compression': 'none'})
node1.start(wait_for_binary_proto=True, wait_other_notice=True)
sessions = []
session = self.patient_exclusive_cql_connection(node1, protocol_version=self.protocol_version)
session.set_keyspace('ks')
sessions.append((True, session))
# open a second session with the node on the old version
session = self.patient_exclusive_cql_connection(node2, protocol_version=self.protocol_version)
session.set_keyspace('ks')
sessions.append((False, session))
if self.CL:
for is_upgraded, session in sessions:
session.default_consistency_level = self.CL
# Let the nodes settle briefly before yielding connections in turn (on the upgraded and non-upgraded alike)
# CASSANDRA-11396 was the impetus for this change, wherein some apparent perf noise was preventing
# CL.ALL from being reached. The newly upgraded node needs to settle because it has just barely started, and each
# non-upgraded node needs a chance to settle as well, because the entire cluster (or isolated nodes) may have been doing resource intensive activities
# immediately before.
for s in sessions:
time.sleep(5)
yield s
def verify_output(query, expected):
output, err = self.run_cqlsh(
node1, query, ['-u', 'cassandra', '-p', 'cassandra'])
if common.is_win():
output = output.replace('\r', '')
self.assertTrue(
expected in output,
"Output \n {%s} \n doesn't contain expected\n {%s}" %
(output, expected))
def tearDown(self):
try:
if self.tempfile:
if is_win():
self.tempfile.close()
os.unlink(self.tempfile.name)
except AttributeError:
pass
super(CqlshCopyTest, self).tearDown()
def tearDown(self):
# Ignore errors before upgrade on Windows
# We ignore errors from 2.1, because windows 2.1
# support is only beta. There are frequent log errors,
# related to filesystem interactions that are a direct result
# of the lack of full functionality on 2.1 Windows, and we dont
# want these to pollute our results.
if is_win() and self.cluster.version() <= '2.2':
self.cluster.nodelist()[1].mark_log_for_errors()
super(UpgradeTester, self).tearDown()
def tearDown(self):
# Ignore errors before upgrade on Windows
# We ignore errors from 2.1, because windows 2.1
# support is only beta. There are frequent log errors,
# related to filesystem interactions that are a direct result
# of the lack of full functionality on 2.1 Windows, and we dont
# want these to pollute our results.
if is_win() and self.cluster.version() <= '2.2':
self.cluster.nodelist()[1].mark_log_for_errors()
super(UpgradeTester, self).tearDown()
def tearDown(self):
try:
if self.tempfile:
if is_win():
self.tempfile.close()
os.unlink(self.tempfile.name)
except AttributeError:
pass
super(CqlshCopyTest, self).tearDown()
def _update_pid(self, process):
pidfile = os.path.join(self.get_path(), 'cassandra.pid')
try:
with open(pidfile, 'r') as f:
if common.is_win() and self.cluster.version() >= '2.1':
self.pid = int(f.readline().strip().decode('utf-16'))
else:
self.pid = int(f.readline().strip())
except IOError:
raise NodeError('Problem starting node %s' % self.name, process)
self.__update_status()
def _update_pid(self, process):
pidfile = os.path.join(self.get_path(), "cassandra.pid")
try:
with open(pidfile, "r") as f:
if common.is_win() and self.cluster.version() >= "2.1":
self.pid = int(f.readline().strip().decode("utf-16"))
else:
self.pid = int(f.readline().strip())
except IOError:
raise NodeError("Problem starting node %s" % self.name, process)
self.__update_status()
def _update_pid(self, process):
pidfile = os.path.join(self.get_path(), 'cassandra.pid')
try:
with open(pidfile, 'r') as f:
if common.is_win() and self.cluster.version() >= '2.1':
self.pid = int(f.readline().strip().decode('utf-16'))
else:
self.pid = int(f.readline().strip())
except IOError:
raise NodeError('Problem starting node %s' % self.name, process)
self.__update_status()
def _update_log4j(self):
super(DseNode, self)._update_log4j()
conf_file = os.path.join(self.get_conf_dir(), common.LOG4J_CONF)
append_pattern = 'log4j.appender.V.File='
log_file = os.path.join(self.get_path(), 'logs', 'solrvalidation.log')
if common.is_win():
log_file = re.sub("\\\\", "/", log_file)
common.replace_in_file(conf_file, append_pattern, append_pattern + log_file)
append_pattern = 'log4j.appender.A.File='
log_file = os.path.join(self.get_path(), 'logs', 'audit.log')
if common.is_win():
log_file = re.sub("\\\\", "/", log_file)
common.replace_in_file(conf_file, append_pattern, append_pattern + log_file)
append_pattern = 'log4j.appender.B.File='
log_file = os.path.join(self.get_path(), 'logs', 'audit', 'dropped-events.log')
if common.is_win():
log_file = re.sub("\\\\", "/", log_file)
common.replace_in_file(conf_file, append_pattern, append_pattern + log_file)
def call_token_generator(self, install_dir, randomPart, nodes):
executable = os.path.join(install_dir, 'tools', 'bin',
'token-generator')
if common.is_win():
executable += ".bat"
args = [executable]
if randomPart is not None:
if randomPart:
args.append("--random")
else:
args.append("--murmur3")
for n in nodes:
args.append(str(n))
debug('Invoking {}'.format(args))
token_gen_output = subprocess.check_output(args)
lines = token_gen_output.split("\n")
dc_tokens = None
generated_tokens = []
for line in lines:
if line.startswith("DC #"):
if dc_tokens is not None:
self.assertGreater(
dc_tokens.__len__(), 0,
"dc_tokens is empty from token-generator {}".format(
args))
generated_tokens.append(dc_tokens)
dc_tokens = []
else:
if line:
m = parse.search('Node #{node_num:d}:{:s}{node_token:d}',
line)
self.assertIsNotNone(
m,
"Line \"{}\" does not match pattern from token-generator {}"
.format(line, args))
node_num = int(m.named['node_num'])
node_token = int(m.named['node_token'])
dc_tokens.append(node_token)
self.assertEqual(
node_num, dc_tokens.__len__(),
"invalid token count from token-generator {}".format(
args))
self.assertIsNotNone(dc_tokens,
"No tokens from token-generator {}".format(args))
self.assertGreater(dc_tokens.__len__(), 0,
"No tokens from token-generator {}".format(args))
generated_tokens.append(dc_tokens)
return generated_tokens
def do_upgrade(self, session):
"""
Upgrades the first node in the cluster and returns a list of
(is_upgraded, Session) tuples. If `is_upgraded` is true, the
Session is connected to the upgraded node.
"""
session.cluster.shutdown()
node1 = self.cluster.nodelist()[0]
node2 = self.cluster.nodelist()[1]
# stop the nodes
node1.drain()
node1.stop(gently=True)
# Ignore errors before upgrade on Windows
# We ignore errors from 2.1, because windows 2.1
# support is only beta. There are frequent log errors,
# related to filesystem interactions that are a direct result
# of the lack of full functionality on 2.1 Windows, and we dont
# want these to pollute our results.
if is_win() and self.cluster.version() <= '2.2':
node1.mark_log_for_errors()
debug('upgrading node1 to {}'.format(self.UPGRADE_PATH.upgrade_version))
node1.set_install_dir(version=self.UPGRADE_PATH.upgrade_version)
# this is a bandaid; after refactoring, upgrades should account for protocol version
new_version_from_build = get_version_from_build(node1.get_install_dir())
if (new_version_from_build >= '3' and self.protocol_version is not None and self.protocol_version < 3):
self.skip('Protocol version {} incompatible '
'with Cassandra version {}'.format(self.protocol_version, new_version_from_build))
node1.set_log_level("DEBUG" if DEBUG else "INFO")
node1.set_configuration_options(values={'internode_compression': 'none'})
node1.start(wait_for_binary_proto=True, wait_other_notice=True)
sessions = []
session = self.patient_exclusive_cql_connection(node1, protocol_version=self.protocol_version)
session.set_keyspace('ks')
sessions.append((True, session))
# open a second session with the node on the old version
session = self.patient_exclusive_cql_connection(node2, protocol_version=self.protocol_version)
session.set_keyspace('ks')
sessions.append((False, session))
if self.CL:
for is_upgraded, session in sessions:
session.default_consistency_level = self.CL
return sessions
def remove_perf_disable_shared_mem(node):
"""
The Jolokia agent is incompatible with the -XX:+PerfDisableSharedMem JVM
option (see https://github.com/rhuss/jolokia/issues/198 for details). This
edits cassandra-env.sh (or the Windows equivalent) to remove that option.
"""
if common.is_win() and node.get_base_cassandra_version() >= 2.1:
conf_file = os.path.join(node.get_conf_dir(), common.CASSANDRA_WIN_ENV)
else:
conf_file = os.path.join(node.get_conf_dir(), common.CASSANDRA_ENV)
pattern = 'PerfDisableSharedMem'
replacement = ''
common.replace_in_file(conf_file, pattern, replacement)
def remove_perf_disable_shared_mem(node):
"""
The Jolokia agent is incompatible with the -XX:+PerfDisableSharedMem JVM
option (see https://github.com/rhuss/jolokia/issues/198 for details). This
edits cassandra-env.sh (or the Windows equivalent) to remove that option.
"""
if common.is_win() and node.get_base_cassandra_version() >= 2.1:
conf_file = os.path.join(node.get_conf_dir(), common.CASSANDRA_WIN_ENV)
else:
conf_file = os.path.join(node.get_conf_dir(), common.CASSANDRA_ENV)
pattern = 'PerfDisableSharedMem'
replacement = ''
common.replace_in_file(conf_file, pattern, replacement)
def stop(self, wait=True, wait_other_notice=False, gently=True):
"""
Stop the node.
- wait: if True (the default), wait for the Cassandra process to be
really dead. Otherwise return after having sent the kill signal.
- wait_other_notice: return only when the other live nodes of the
cluster have marked this node has dead.
- gently: Let Cassandra clean up and shut down properly. Otherwise do
a 'kill -9' which shuts down faster.
"""
if self.is_running():
if wait_other_notice:
# tstamp = time.time()
marks = [
(node, node.mark_log())
for node in list(self.cluster.nodes.values())
if node.is_running() and node is not self
]
if common.is_win():
self.stop_win(wait, wait_other_notice, gently)
else:
if gently:
os.kill(self.pid, signal.SIGTERM)
else:
os.kill(self.pid, signal.SIGKILL)
if wait_other_notice:
for node, mark in marks:
node.watch_log_for_death(self, from_mark=mark)
# print node.name, "has marked", self.name, "down in " + str(time.time() - tstamp) + "s"
else:
time.sleep(0.1)
still_running = self.is_running()
if still_running and wait:
wait_time_sec = 1
for i in xrange(0, 7):
# we'll double the wait time each try and cassandra should
# not take more than 1 minute to shutdown
time.sleep(wait_time_sec)
if not self.is_running():
return True
wait_time_sec = wait_time_sec * 2
raise NodeError("Problem stopping node %s" % self.name)
else:
return True
else:
return False
def call_token_generator(self, install_dir, randomPart, nodes):
executable = install_dir + "/tools/bin/token-generator"
if common.is_win():
executable += ".bat"
args = [executable]
if randomPart is not None:
if randomPart:
args.append("--random")
else:
args.append("--murmur3")
for n in nodes:
args.append(str(n))
debug('Invoking %s' % (args, ))
token_gen_output = subprocess.check_output(args)
lines = token_gen_output.split("\n")
dc_tokens = None
generated_tokens = []
for line in lines:
if line.startswith("DC #"):
if dc_tokens is not None:
self.assertGreater(
dc_tokens.__len__(), 0,
"dc_tokens is empty from token-generator %r" % args)
generated_tokens.append(dc_tokens)
dc_tokens = []
else:
if line.__len__() > 0:
m = re.search("^ Node #(\d+): [ ]*([-]?\d+)$", line)
self.assertIsNotNone(
m,
"Line \"%r\" does not match pattern from token-generator %r"
% (line, args))
node_num = int(m.group(1))
node_token = int(m.group(2))
dc_tokens.append(node_token)
self.assertEqual(
node_num, dc_tokens.__len__(),
"invalid token count from token-generator %r" % args)
self.assertIsNotNone(dc_tokens,
"No tokens from token-generator %r" % args)
self.assertGreater(dc_tokens.__len__(), 0,
"No tokens from token-generator %r" % args)
generated_tokens.append(dc_tokens)
return generated_tokens
def test_ignore_failure_policy(self):
"""
Test the ignore commitlog failure policy
"""
self.prepare(configuration={
'commit_failure_policy': 'ignore'
})
self._provoke_commitlog_failure()
failure = self.node1.grep_log(r"ERROR \[COMMIT-LOG-ALLOCATOR\].+Failed .+ commit log segments")
assert failure, "Cannot find the commitlog failure message in logs"
assert self.node1.is_running(), "Node1 should still be running"
# on Windows, we can't delete the segments if they're chmod to 0 so they'll still be available for use by CLSM,
# and we can still create new segments since os.chmod is limited to stat.S_IWRITE and stat.S_IREAD to set files
# as read-only. New mutations will still be allocated and WriteTimeouts will not be raised. It's sufficient that
# we confirm that a) the node isn't dead (stop) and b) the node doesn't terminate the thread (stop_commit)
query = "INSERT INTO test (key, col1) VALUES (2, 2);"
if is_win():
# We expect this to succeed
self.session1.execute(query)
assert not self.node1.grep_log("terminating thread"), "thread was terminated but CL error should have been ignored."
assert self.node1.is_running(), "Node1 should still be running after an ignore error on CL"
else:
with pytest.raises((OperationTimedOut, WriteTimeout)):
self.session1.execute(query)
# Should not exist
assert_none(self.session1, "SELECT * FROM test where key=2;")
# bring back the node commitlogs
self._change_commitlog_perms(stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC)
self.session1.execute("""
INSERT INTO test (key, col1) VALUES (3, 3);
""")
assert_one(
self.session1,
"SELECT * FROM test where key=3;",
[3, 3]
)
time.sleep(2)
assert_one(
self.session1,
"SELECT * FROM test where key=2;",
[2, 2]
)
def ignore_failure_policy_test(self):
"""
Test the ignore commitlog failure policy
"""
self.prepare(configuration={
'commit_failure_policy': 'ignore'
})
self._provoke_commitlog_failure()
failure = self.node1.grep_log("ERROR \[COMMIT-LOG-ALLOCATOR\].+Failed .+ commit log segments")
self.assertTrue(failure, "Cannot find the commitlog failure message in logs")
self.assertTrue(self.node1.is_running(), "Node1 should still be running")
# on Windows, we can't delete the segments if they're chmod to 0 so they'll still be available for use by CLSM,
# and we can still create new segments since os.chmod is limited to stat.S_IWRITE and stat.S_IREAD to set files
# as read-only. New mutations will still be allocated and WriteTimeouts will not be raised. It's sufficient that
# we confirm that a) the node isn't dead (stop) and b) the node doesn't terminate the thread (stop_commit)
query = "INSERT INTO test (key, col1) VALUES (2, 2);"
if is_win():
# We expect this to succeed
self.session1.execute(query)
self.assertFalse(self.node1.grep_log("terminating thread"), "thread was terminated but CL error should have been ignored.")
self.assertTrue(self.node1.is_running(), "Node1 should still be running after an ignore error on CL")
else:
with self.assertRaises((OperationTimedOut, WriteTimeout)):
self.session1.execute(query)
# Should not exist
assert_none(self.session1, "SELECT * FROM test where key=2;")
# bring back the node commitlogs
self._change_commitlog_perms(stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC)
self.session1.execute("""
INSERT INTO test (key, col1) VALUES (3, 3);
""")
assert_one(
self.session1,
"SELECT * FROM test where key=3;",
[3, 3]
)
time.sleep(2)
assert_one(
self.session1,
"SELECT * FROM test where key=2;",
[2, 2]
)
def force_repair_async_1_test(self, ):
"""
test forceRepairAsync(String keyspace, boolean isSequential,
Collection<String> dataCenters,
Collection<String> hosts,
boolean primaryRange, boolean fullRepair, String... columnFamilies)
"""
opt = self._deprecated_repair_jmx("forceRepairAsync(java.lang.String,boolean,java.util.Collection,java.util.Collection,boolean,boolean,[Ljava.lang.String;)",
['ks', True, [], [], False, False, ["cf"]])
self.assertEqual(opt["parallelism"], "parallel" if is_win() else "sequential", opt)
self.assertEqual(opt["primary_range"], "false", opt)
self.assertEqual(opt["incremental"], "true", opt)
self.assertEqual(opt["job_threads"], "1", opt)
self.assertEqual(opt["data_centers"], "[]", opt)
self.assertEqual(opt["hosts"], "[]", opt)
self.assertEqual(opt["column_families"], "[cf]", opt)
def stress(self, stress_options):
stress = common.get_stress_bin(self.get_cassandra_dir())
livenodes = [ node.network_interfaces['storage'][0] for node in list(self.nodes.values()) if node.is_live() ]
if len(livenodes) == 0:
print_("No live node")
return
args = [ stress, '-d', ",".join(livenodes) ] + stress_options
try:
# need to set working directory for env on Windows
if common.is_win():
subprocess.call(args, cwd=common.parse_path(stress))
else:
subprocess.call(args)
except KeyboardInterrupt:
pass
return self
def force_repair_range_async_3_test(self, ):
"""
test forceRepairRangeAsync(String beginToken, String endToken,
String keyspaceName, boolean isSequential,
boolean isLocal, boolean fullRepair,
String... columnFamilies)
"""
opt = self._deprecated_repair_jmx("forceRepairRangeAsync(java.lang.String,java.lang.String,java.lang.String,boolean,boolean,boolean,[Ljava.lang.String;)",
["0", "1000", "ks", True, True, True, ["cf"]])
self.assertEqual(opt["parallelism"], "parallel" if is_win() else "sequential", opt)
self.assertEqual(opt["primary_range"], "false", opt)
self.assertEqual(opt["incremental"], "false", opt)
self.assertEqual(opt["job_threads"], "1", opt)
self.assertEqual(opt["data_centers"], "[dc1]", opt)
self.assertEqual(opt["hosts"], "[]", opt)
self.assertEqual(opt["ranges"], "1", opt)
self.assertEqual(opt["column_families"], "[cf]", opt)
def find_libjemalloc():
if is_win():
# let the normal bat script handle finding libjemalloc
return ""
this_dir = os.path.dirname(os.path.realpath(__file__))
script = os.path.join(this_dir, "findlibjemalloc.sh")
try:
p = subprocess.Popen([script], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
if stderr or not stdout:
return "-" # tells C* not to look for libjemalloc
else:
return stdout
except Exception as exc:
print "Failed to run script to prelocate libjemalloc ({}): {}".format(script, exc)
return ""
def force_repair_range_async_2_test(self, ):
"""
test forceRepairRangeAsync(String beginToken, String endToken,
String keyspaceName, int parallelismDegree,
Collection<String> dataCenters,
Collection<String> hosts,
boolean fullRepair, String... columnFamilies)
"""
opt = self._deprecated_repair_jmx("forceRepairRangeAsync(java.lang.String,java.lang.String,java.lang.String,int,java.util.Collection,java.util.Collection,boolean,[Ljava.lang.String;)",
["0", "1000", "ks", 2, [], [], True, ["cf"]])
self.assertEqual(opt["parallelism"], "parallel" if is_win() else "dc_parallel", opt)
self.assertEqual(opt["primary_range"], "false", opt)
self.assertEqual(opt["incremental"], "false", opt)
self.assertEqual(opt["job_threads"], "1", opt)
self.assertEqual(opt["data_centers"], "[]", opt)
self.assertEqual(opt["hosts"], "[]", opt)
self.assertEqual(opt["ranges"], "1", opt)
self.assertEqual(opt["column_families"], "[cf]", opt)
def test_force_repair_range_async_1(self):
"""
test forceRepairRangeAsync(String beginToken, String endToken,
String keyspaceName, boolean isSequential,
Collection<String> dataCenters,
Collection<String> hosts, boolean fullRepair,
String... columnFamilies)
"""
opt = self._deprecated_repair_jmx("forceRepairRangeAsync(java.lang.String,java.lang.String,java.lang.String,boolean,java.util.Collection,java.util.Collection,boolean,[Ljava.lang.String;)",
["0", "1000", "ks", True, ["dc1"], [], False, ["cf"]])
assert opt["parallelism"], "parallel" if is_win() else "sequential" == opt
assert opt["primary_range"], "false" == opt
assert opt["incremental"], "true" == opt
assert opt["job_threads"], "1" == opt
assert opt["data_centers"], "[dc1]" == opt
assert opt["hosts"], "[]" == opt
assert opt["ranges"], "1" == opt
assert opt["column_families"], "[cf]" == opt
