def use_cluster(cluster_name, nodes, ipformat=None, start=True):
if is_current_cluster(cluster_name, nodes):
log.debug("Using existing cluster %s", cluster_name)
return
global CCM_CLUSTER
if CCM_CLUSTER:
log.debug("Stopping cluster %s", CCM_CLUSTER.name)
CCM_CLUSTER.stop()
try:
try:
cluster = CCMClusterFactory.load(path, cluster_name)
log.debug("Found existing ccm %s cluster; clearing", cluster_name)
cluster.clear()
cluster.set_install_dir(**CCM_KWARGS)
except Exception:
log.debug("Creating new ccm %s cluster with %s", cluster_name,
CCM_KWARGS)
cluster = CCMCluster(path, cluster_name, **CCM_KWARGS)
cluster.set_configuration_options({'start_native_transport': True})
common.switch_cluster(path, cluster_name)
cluster.populate(nodes, ipformat=ipformat)
if start:
log.debug("Starting ccm %s cluster", cluster_name)
cluster.start(wait_for_binary_proto=True, wait_other_notice=True)
setup_test_keyspace()
CCM_CLUSTER = cluster
except Exception:
log.exception("Failed to start ccm cluster:")
raise
def clear_and_use_multidc(dc_list):
teardown_package()
try:
try:
cluster = CCMCluster.load(path, MULTIDC_CLUSTER_NAME)
log.debug("Found existing ccm test multi-dc cluster, clearing")
cluster.clear()
except Exception:
log.debug("Creating new ccm test multi-dc cluster")
if CASSANDRA_DIR:
cluster = CCMCluster(path, MULTIDC_CLUSTER_NAME, cassandra_dir=CASSANDRA_DIR)
else:
cluster = CCMCluster(path, MULTIDC_CLUSTER_NAME, cassandra_version=CASSANDRA_VERSION)
cluster.set_configuration_options({'start_native_transport': True})
common.switch_cluster(path, MULTIDC_CLUSTER_NAME)
cluster.populate(dc_list)
log.debug("Starting ccm test cluster")
cluster.start(wait_for_binary_proto=True, wait_other_notice=True)
except Exception:
log.exception("Failed to start ccm cluster:")
raise
global CCM_CLUSTER
CCM_CLUSTER = cluster
setup_test_keyspace()
log.debug("Switched to multidc cluster")
def use_multidc(dc_list):
teardown_package()
try:
try:
cluster = CCMCluster.load(path, MULTIDC_CLUSTER_NAME)
log.debug("Found existing ccm test multi-dc cluster, clearing")
cluster.clear()
except Exception:
log.debug("Creating new ccm test multi-dc cluster")
if CASSANDRA_DIR:
cluster = CCMCluster(path,
MULTIDC_CLUSTER_NAME,
cassandra_dir=CASSANDRA_DIR)
else:
cluster = CCMCluster(path,
MULTIDC_CLUSTER_NAME,
cassandra_version=CASSANDRA_VERSION)
cluster.set_configuration_options({'start_native_transport': True})
common.switch_cluster(path, MULTIDC_CLUSTER_NAME)
cluster.populate(dc_list)
log.debug("Starting ccm test cluster")
cluster.start(wait_for_binary_proto=True, wait_other_notice=True)
except Exception:
log.exception("Failed to start ccm cluster:")
raise
global CCM_CLUSTER
CCM_CLUSTER = cluster
setup_test_keyspace()
log.debug("Switched to multidc cluster")
def run(self):
try:
cluster = Cluster(self.path, self.name, cassandra_dir=self.options.cassandra_dir, cassandra_version=self.options.cassandra_version, verbose=True)
except OSError as e:
cluster_dir = os.path.join(self.path, self.name)
import traceback
print >> sys.stderr, 'Cannot create cluster: %s\n%s' % (str(e), traceback.format_exc())
exit(1)
if self.options.partitioner:
cluster.set_partitioner(self.options.partitioner)
if cluster.version() >= "1.2" and self.options.binary_protocol:
cluster.set_configuration_options({ 'start_native_transport' : True })
if cluster.version() >= "1.2" and self.options.vnodes:
cluster.set_configuration_options({ 'num_tokens' : 256 })
if not self.options.no_switch:
common.switch_cluster(self.path, self.name)
print 'Current cluster is now: %s' % self.name
if self.nodes is not None:
try:
if self.options.debug_log:
cluster.set_log_level("DEBUG")
if self.options.trace_log:
cluster.set_log_level("TRACE")
cluster.populate(self.nodes, use_vnodes=self.options.vnodes, ipprefix=self.options.ipprefix)
if self.options.start_nodes:
cluster.start(verbose=self.options.debug, wait_for_binary_proto=self.options.binary_protocol)
except common.ArgumentError as e:
print >> sys.stderr, str(e)
exit(1)
def setup_module(cls):
validate_ccm_viable()
validate_host_viable()
cls.ccm_cluster = object()
teardown_package()
try:
try:
ccm_cluster = CCMCluster.load(path, IPV6_CLUSTER_NAME)
log.debug("Found existing ccm test ipv6 cluster, clearing")
ccm_cluster.clear()
except Exception:
log.debug("Creating new ccm test ipv6 cluster")
if CASSANDRA_DIR:
ccm_cluster = CCMCluster(path, IPV6_CLUSTER_NAME, cassandra_dir=CASSANDRA_DIR)
else:
ccm_cluster = CCMCluster(path, IPV6_CLUSTER_NAME, cassandra_version=CASSANDRA_VERSION)
ccm_cluster.set_configuration_options({'start_native_transport': True})
common.switch_cluster(path, IPV6_CLUSTER_NAME)
ccm_cluster.populate(1, ipformat='::%d')
log.debug("Starting ccm test cluster")
ccm_cluster.start(wait_for_binary_proto=True)
except Exception:
log.exception("Failed to start ccm cluster:")
raise
log.debug("Switched to ipv6 cluster")
cls.ccm_cluster = ccm_cluster
def run(self):
try:
cluster = Cluster(self.path, self.name, cassandra_dir=self.options.cassandra_dir, cassandra_version=self.options.cassandra_version, verbose=True)
except OSError as e:
cluster_dir = os.path.join(self.path, self.name)
print >> sys.stderr, 'Cannot create cluster: %s' % str(e)
exit(1)
if self.options.partitioner:
cluster.set_partitioner(self.options.partitioner)
if cluster.version() >= "1.2" and self.options.binary_protocol:
cluster.set_configuration_options({ 'start_native_transport' : True })
if not self.options.no_switch:
common.switch_cluster(self.path, self.name)
print 'Current cluster is now: %s' % self.name
if self.nodes is not None:
try:
cluster.set_log_level("DEBUG")
cluster.populate(self.nodes)
if self.options.start_nodes:
cluster.start(verbose=self.options.debug)
except common.ArgumentError as e:
print >> sys.stderr, str(e)
exit(1)
def setup_package():
print_('Using Cassandra version: %s' % CASSANDRA_VERSION)
try:
try:
cluster = CCMCluster.load(path, CLUSTER_NAME)
log.debug("Found existing ccm test cluster, clearing")
cluster.clear()
if CASSANDRA_DIR:
cluster.set_cassandra_dir(cassandra_dir=CASSANDRA_DIR)
else:
cluster.set_cassandra_dir(cassandra_version=CASSANDRA_VERSION)
except Exception:
if CASSANDRA_DIR:
log.debug("Creating new ccm test cluster with cassandra dir %s", CASSANDRA_DIR)
cluster = CCMCluster(path, CLUSTER_NAME, cassandra_dir=CASSANDRA_DIR)
else:
log.debug("Creating new ccm test cluster with version %s", CASSANDRA_VERSION)
cluster = CCMCluster(path, CLUSTER_NAME, cassandra_version=CASSANDRA_VERSION)
cluster.set_configuration_options({'start_native_transport': True})
common.switch_cluster(path, CLUSTER_NAME)
cluster.populate(3)
log.debug("Starting ccm test cluster")
cluster.start(wait_for_binary_proto=True, wait_other_notice=True)
except Exception:
log.exception("Failed to start ccm cluster:")
raise
global CCM_CLUSTER
CCM_CLUSTER = cluster
setup_test_keyspace()
def setup_module(cls):
validate_ccm_viable()
validate_host_viable()
cls.ccm_cluster = object()
teardown_package()
try:
try:
ccm_cluster = CCMCluster.load(path, IPV6_CLUSTER_NAME)
log.debug("Found existing ccm test ipv6 cluster, clearing")
ccm_cluster.clear()
except Exception:
log.debug("Creating new ccm test ipv6 cluster")
if CASSANDRA_DIR:
ccm_cluster = CCMCluster(path,
IPV6_CLUSTER_NAME,
cassandra_dir=CASSANDRA_DIR)
else:
ccm_cluster = CCMCluster(path,
IPV6_CLUSTER_NAME,
cassandra_version=CASSANDRA_VERSION)
ccm_cluster.set_configuration_options(
{'start_native_transport': True})
common.switch_cluster(path, IPV6_CLUSTER_NAME)
ccm_cluster.populate(1, ipformat='::%d')
log.debug("Starting ccm test cluster")
ccm_cluster.start(wait_for_binary_proto=True)
except Exception:
log.exception("Failed to start ccm cluster:")
raise
log.debug("Switched to ipv6 cluster")
cls.ccm_cluster = ccm_cluster
def setup_package():
print_('Using Cassandra version: %s' % CASSANDRA_VERSION)
try:
try:
cluster = CCMCluster.load(path, CLUSTER_NAME)
log.debug("Found existing ccm test cluster, clearing")
cluster.clear()
cluster.set_cassandra_dir(cassandra_version=CASSANDRA_VERSION)
except Exception:
log.debug("Creating new ccm test cluster with version %s",
CASSANDRA_VERSION)
cluster = CCMCluster(path,
CLUSTER_NAME,
cassandra_version=CASSANDRA_VERSION)
cluster.set_configuration_options({'start_native_transport': True})
common.switch_cluster(path, CLUSTER_NAME)
cluster.populate(3)
log.debug("Starting ccm test cluster")
cluster.start(wait_for_binary_proto=True)
except Exception:
log.exception("Failed to start ccm cluster:")
raise
global CCM_CLUSTER
CCM_CLUSTER = cluster
setup_test_keyspace()
def use_cluster(cluster_name, nodes, ipformat=None, start=True):
global CCM_CLUSTER
if USE_CASS_EXTERNAL:
if CCM_CLUSTER:
log.debug("Using external CCM cluster {0}".format(CCM_CLUSTER.name))
else:
log.debug("Using unnamed external cluster")
return
if is_current_cluster(cluster_name, nodes):
log.debug("Using existing cluster, matching topology: {0}".format(cluster_name))
else:
if CCM_CLUSTER:
log.debug("Stopping existing cluster, topology mismatch: {0}".format(CCM_CLUSTER.name))
CCM_CLUSTER.stop()
try:
CCM_CLUSTER = CCMClusterFactory.load(path, cluster_name)
log.debug("Found existing CCM cluster, {0}; clearing.".format(cluster_name))
CCM_CLUSTER.clear()
CCM_CLUSTER.set_install_dir(**CCM_KWARGS)
except Exception:
ex_type, ex, tb = sys.exc_info()
log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb)))
del tb
log.debug("Creating new CCM cluster, {0}, with args {1}".format(cluster_name, CCM_KWARGS))
if DSE_VERSION:
log.error("creating dse cluster")
CCM_CLUSTER = DseCluster(path, cluster_name, **CCM_KWARGS)
else:
CCM_CLUSTER = CCMCluster(path, cluster_name, **CCM_KWARGS)
CCM_CLUSTER.set_configuration_options({'start_native_transport': True})
if CASSANDRA_VERSION >= '2.2':
CCM_CLUSTER.set_configuration_options({'enable_user_defined_functions': True})
if CASSANDRA_VERSION >= '3.0':
CCM_CLUSTER.set_configuration_options({'enable_scripted_user_defined_functions': True})
common.switch_cluster(path, cluster_name)
CCM_CLUSTER.populate(nodes, ipformat=ipformat)
try:
jvm_args = []
# This will enable the Mirroring query handler which will echo our custom payload k,v pairs back
if PROTOCOL_VERSION >= 4:
jvm_args = [" -Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler"]
if start:
log.debug("Starting CCM cluster: {0}".format(cluster_name))
CCM_CLUSTER.start(wait_for_binary_proto=True, wait_other_notice=True, jvm_args=jvm_args)
setup_keyspace(ipformat=ipformat)
except Exception:
log.exception("Failed to start CCM cluster; removing cluster.")
if os.name == "nt":
if CCM_CLUSTER:
for node in CCM_CLUSTER.nodes.itervalues():
os.system("taskkill /F /PID " + str(node.pid))
else:
call(["pkill", "-9", "-f", ".ccm"])
remove_cluster()
raise
def use_cluster(cluster_name, nodes, ipformat=None, start=True):
if is_current_cluster(cluster_name, nodes):
log.debug("Using existing cluster %s", cluster_name)
return
global CCM_CLUSTER
if CCM_CLUSTER:
log.debug("Stopping cluster %s", CCM_CLUSTER.name)
CCM_CLUSTER.stop()
try:
try:
cluster = CCMClusterFactory.load(path, cluster_name)
log.debug("Found existing ccm %s cluster; clearing", cluster_name)
cluster.clear()
cluster.set_install_dir(**CCM_KWARGS)
except Exception:
log.debug("Creating new ccm %s cluster with %s", cluster_name, CCM_KWARGS)
cluster = CCMCluster(path, cluster_name, **CCM_KWARGS)
cluster.set_configuration_options({'start_native_transport': True})
common.switch_cluster(path, cluster_name)
cluster.populate(nodes, ipformat=ipformat)
if start:
log.debug("Starting ccm %s cluster", cluster_name)
cluster.start(wait_for_binary_proto=True, wait_other_notice=True)
setup_test_keyspace()
CCM_CLUSTER = cluster
except Exception:
log.exception("Failed to start ccm cluster:")
raise
def run(self):
try:
if self.options.dse or (not self.options.version and common.isDse(self.options.install_dir)):
cluster = DseCluster(self.path, self.name, install_dir=self.options.install_dir, version=self.options.version, dse_username=self.options.dse_username, dse_password=self.options.dse_password, dse_credentials_file=self.options.dse_credentials_file, opscenter=self.options.opscenter, verbose=True)
else:
cluster = Cluster(self.path, self.name, install_dir=self.options.install_dir, version=self.options.version, verbose=True)
except OSError as e:
import traceback
print_('Cannot create cluster: %s\n%s' % (str(e), traceback.format_exc()), file=sys.stderr)
exit(1)
if self.options.partitioner:
cluster.set_partitioner(self.options.partitioner)
if cluster.cassandra_version() >= "1.2.5":
self.options.binary_protocol = True
if self.options.binary_protocol:
cluster.set_configuration_options({'start_native_transport': True})
if cluster.cassandra_version() >= "1.2" and self.options.vnodes:
cluster.set_configuration_options({'num_tokens': 256})
if not self.options.no_switch:
common.switch_cluster(self.path, self.name)
print_('Current cluster is now: %s' % self.name)
if not (self.options.ipprefix or self.options.ipformat):
self.options.ipformat = '127.0.0.%d'
if self.options.ssl_path:
cluster.enable_ssl(self.options.ssl_path, self.options.require_client_auth)
if self.options.node_ssl_path:
cluster.enable_internode_ssl(self.options.node_ssl_path)
if self.options.datadirs:
cluster.set_datadir_count(self.options.datadirs)
if self.nodes is not None:
try:
if self.options.debug_log:
cluster.set_log_level("DEBUG")
if self.options.trace_log:
cluster.set_log_level("TRACE")
cluster.populate(self.nodes, self.options.debug, use_vnodes=self.options.vnodes, ipprefix=self.options.ipprefix, ipformat=self.options.ipformat, install_byteman=self.options.install_byteman)
if self.options.start_nodes:
profile_options = None
if self.options.profile:
profile_options = {}
if self.options.profile_options:
profile_options['options'] = self.options.profile_options
if cluster.start(verbose=self.options.debug_log, wait_for_binary_proto=self.options.binary_protocol, jvm_args=self.options.jvm_args, profile_options=profile_options, allow_root=self.options.allow_root) is None:
details = ""
if not self.options.debug_log:
details = " (you can use --debug-log for more information)"
print_("Error starting nodes, see above for details%s" % details, file=sys.stderr)
except common.ArgumentError as e:
print_(str(e), file=sys.stderr)
exit(1)
def run(self):
try:
if self.options.dse or (not self.options.version and common.isDse(self.options.install_dir)):
cluster = DseCluster(self.path, self.name, install_dir=self.options.install_dir, version=self.options.version, dse_username=self.options.dse_username, dse_password=self.options.dse_password, opscenter=self.options.opscenter, verbose=True)
else:
cluster = Cluster(self.path, self.name, install_dir=self.options.install_dir, version=self.options.version, verbose=True)
except OSError as e:
cluster_dir = os.path.join(self.path, self.name)
import traceback
print_('Cannot create cluster: %s\n%s' % (str(e), traceback.format_exc()), file=sys.stderr)
exit(1)
if self.options.partitioner:
cluster.set_partitioner(self.options.partitioner)
if cluster.cassandra_version() >= "1.2.5":
self.options.binary_protocol = True
if self.options.binary_protocol:
cluster.set_configuration_options({ 'start_native_transport' : True })
if cluster.cassandra_version() >= "1.2" and self.options.vnodes:
cluster.set_configuration_options({ 'num_tokens' : 256 })
if not self.options.no_switch:
common.switch_cluster(self.path, self.name)
print_('Current cluster is now: %s' % self.name)
if not (self.options.ipprefix or self.options.ipformat):
self.options.ipformat = '127.0.0.%d'
if self.options.ssl_path:
cluster.enable_ssl(self.options.ssl_path, self.options.require_client_auth)
if self.options.node_ssl_path:
cluster.enable_internode_ssl(self.options.node_ssl_path)
if self.nodes is not None:
try:
if self.options.debug_log:
cluster.set_log_level("DEBUG")
if self.options.trace_log:
cluster.set_log_level("TRACE")
cluster.populate(self.nodes, self.options.debug, use_vnodes=self.options.vnodes, ipprefix=self.options.ipprefix, ipformat=self.options.ipformat)
if self.options.start_nodes:
profile_options = None
if self.options.profile:
profile_options = {}
if self.options.profile_options:
profile_options['options'] = self.options.profile_options
if cluster.start(verbose=self.options.debug_log, wait_for_binary_proto=self.options.binary_protocol, jvm_args=self.options.jvm_args, profile_options=profile_options) is None:
details = ""
if not self.options.debug_log:
details = " (you can use --debug-log for more information)"
print_("Error starting nodes, see above for details%s" % details, file=sys.stderr)
except common.ArgumentError as e:
print_(str(e), file=sys.stderr)
exit(1)
def run(self):
try:
cluster = Cluster(self.path,
self.name,
cassandra_dir=self.options.cassandra_dir,
cassandra_version=self.options.cassandra_version,
verbose=True)
except OSError as e:
cluster_dir = os.path.join(self.path, self.name)
import traceback
print >> sys.stderr, 'Cannot create cluster: %s\n%s' % (
str(e), traceback.format_exc())
exit(1)
if self.options.partitioner:
cluster.set_partitioner(self.options.partitioner)
if cluster.version() >= "1.2.5":
self.options.binary_protocol = True
if self.options.binary_protocol:
cluster.set_configuration_options({'start_native_transport': True})
if cluster.version() >= "1.2" and self.options.vnodes:
cluster.set_configuration_options({'num_tokens': 256})
if not self.options.no_switch:
common.switch_cluster(self.path, self.name)
print 'Current cluster is now: %s' % self.name
if self.nodes is not None:
try:
if self.options.debug_log:
cluster.set_log_level("DEBUG")
if self.options.trace_log:
cluster.set_log_level("TRACE")
cluster.populate(self.nodes,
use_vnodes=self.options.vnodes,
ipprefix=self.options.ipprefix)
if self.options.start_nodes:
profile_options = None
if self.options.profile:
profile_options = {}
if self.options.profile_options:
profile_options[
'options'] = self.options.profile_options
if cluster.start(
verbose=self.options.debug,
wait_for_binary_proto=self.options.binary_protocol,
jvm_args=self.options.jvm_args,
profile_options=profile_options) is None:
details = ""
if not self.options.debug:
details = " (you can use --debug for more information)"
print >> sys.stderr, "Error starting nodes, see above for details%s" % details
except common.ArgumentError as e:
print >> sys.stderr, str(e)
exit(1)
def setup_class(cls):
"""
Class-level fixture. Called once for all tests.
We create a cluster object here and save it in the provided class instance
Create 2 regular users and a new superuser.
Enable authentication by setting 'authenticator': 'PasswordAuthenticator'.
If invoked from authorization class enable authorization by setting 'authorizer': 'CassandraAuthorizer'.
"""
try:
ccm_cluster = CCMCluster.load(tests.integration.path, AUTH_CLUSTER_NAME)
log.debug("Found existing ccm test authentication cluster, removing")
ccm_cluster.remove()
except Exception:
log.debug("Can not load cluster %s ....." % AUTH_CLUSTER_NAME)
log.debug("Creating new ccm test authentication cluster")
if tests.integration.CASSANDRA_DIR:
ccm_cluster = CCMCluster(tests.integration.path, AUTH_CLUSTER_NAME, cassandra_dir=tests.integration.CASSANDRA_DIR)
else:
ccm_cluster = CCMCluster(tests.integration.path, AUTH_CLUSTER_NAME, cassandra_version=tests.integration.CASSANDRA_VERSION)
ccm_cluster.set_configuration_options({'start_native_transport': True})
ccm_cluster.set_configuration_options({'authenticator': 'PasswordAuthenticator'})
#
# This method is called either with AuthenticationTests class or with AuthorizedAuthenticationTests class.
# In the second case we enable CassandraAuthorizer
#
if cls.__name__ == 'AuthorizedAuthenticationTests':
print "Running tests with Cassandra Authorizer Enabled"
log.info("Running tests with Cassandra Authorizer Enabled")
ccm_cluster.set_configuration_options({'authorizer': 'CassandraAuthorizer'})
else:
print "Running tests with Cassandra Authorizer Disabled"
log.info("Running tests with Cassandra Authorizer Disabled")
common.switch_cluster(tests.integration.path, AUTH_CLUSTER_NAME)
ccm_cluster.populate(1)
log.debug("Starting ccm test authentication cluster")
ccm_cluster.start(wait_for_binary_proto=True)
if not wait_for_cassandra() or not try_connecting('cassandra', 'cassandra'):
log.exception("Can not talk to cassandra")
raise Exception('Can not talk to cassandra')
log.debug("Switched to AUTH_CLUSTER_NAME cluster")
cls.ccm_cluster = ccm_cluster
cls.root_cluster = cls.cluster_as('cassandra', 'cassandra')
cls.root_session = cls.root_cluster.connect()
cls.create_user(cls.root_cluster, cls.test_user, cls.password)
cls.create_user(cls.root_cluster, cls.test_other_user, cls.password)
cls.create_user(cls.root_cluster, cls.suser, cls.password, su=True)
def use_cluster(cluster_name, nodes, ipformat=None, start=True):
global CCM_CLUSTER
if USE_CASS_EXTERNAL:
if CCM_CLUSTER:
log.debug("Using external CCM cluster {0}".format(CCM_CLUSTER.name))
else:
log.debug("Using unnamed external cluster")
return
if is_current_cluster(cluster_name, nodes):
log.debug("Using existing cluster, matching topology: {0}".format(cluster_name))
else:
if CCM_CLUSTER:
log.debug("Stopping existing cluster, topology mismatch: {0}".format(CCM_CLUSTER.name))
CCM_CLUSTER.stop()
try:
CCM_CLUSTER = CCMClusterFactory.load(path, cluster_name)
log.debug("Found existing CCM cluster, {0}; clearing.".format(cluster_name))
CCM_CLUSTER.clear()
CCM_CLUSTER.set_install_dir(**CCM_KWARGS)
except Exception:
ex_type, ex, tb = sys.exc_info()
log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb)))
del tb
log.debug("Creating new CCM cluster, {0}, with args {1}".format(cluster_name, CCM_KWARGS))
CCM_CLUSTER = CCMCluster(path, cluster_name, **CCM_KWARGS)
CCM_CLUSTER.set_configuration_options({'start_native_transport': True})
if CASSANDRA_VERSION >= '2.2':
CCM_CLUSTER.set_configuration_options({'enable_user_defined_functions': True})
common.switch_cluster(path, cluster_name)
CCM_CLUSTER.populate(nodes, ipformat=ipformat)
try:
jvm_args = []
# This will enable the Mirroring query handler which will echo our custom payload k,v pairs back
if PROTOCOL_VERSION >= 4:
jvm_args = [" -Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler"]
if start:
log.debug("Starting CCM cluster: {0}".format(cluster_name))
CCM_CLUSTER.start(wait_for_binary_proto=True, wait_other_notice=True, jvm_args=jvm_args)
setup_keyspace(ipformat=ipformat)
except Exception:
log.exception("Failed to start CCM cluster; removing cluster.")
if os.name == "nt":
if CCM_CLUSTER:
for node in CCM_CLUSTER.nodes.itervalues():
os.system("taskkill /F /PID " + str(node.pid))
else:
call(["pkill", "-9", "-f", ".ccm"])
remove_cluster()
raise
def run(self):
try:
cluster = Cluster(self.path, self.name, cassandra_dir=self.options.cassandra_dir, cassandra_version=self.options.cassandra_version, verbose=True)
except OSError as e:
cluster_dir = os.path.join(self.path, self.name)
import traceback
print_('Cannot create cluster: %s\n%s' % (str(e), traceback.format_exc()), file=sys.stderr)
exit(1)
if self.options.partitioner:
cluster.set_partitioner(self.options.partitioner)
if cluster.version() >= "1.2.5":
self.options.binary_protocol = True
if self.options.binary_protocol:
cluster.set_configuration_options({ 'start_native_transport' : True })
if cluster.version() >= "1.2" and self.options.vnodes:
cluster.set_configuration_options({ 'num_tokens' : 256 })
if not self.options.no_switch:
common.switch_cluster(self.path, self.name)
print_('Current cluster is now: %s' % self.name)
if not (self.options.ipprefix or self.options.ipformat):
self.options.ipformat = '127.0.0.%d'
if self.nodes is not None:
try:
if self.options.debug_log:
cluster.set_log_level("DEBUG")
if self.options.trace_log:
cluster.set_log_level("TRACE")
cluster.populate(self.nodes, use_vnodes=self.options.vnodes, ipprefix=self.options.ipprefix, ipformat=self.options.ipformat)
if self.options.start_nodes:
profile_options = None
if self.options.profile:
profile_options = {}
if self.options.profile_options:
profile_options['options'] = self.options.profile_options
if cluster.start(verbose=self.options.debug, wait_for_binary_proto=self.options.binary_protocol, jvm_args=self.options.jvm_args, profile_options=profile_options) is None:
details = ""
if not self.options.debug:
details = " (you can use --debug for more information)"
print_("Error starting nodes, see above for details%s" % details, file=sys.stderr)
except common.ArgumentError as e:
print_(str(e), file=sys.stderr)
exit(1)
def use_cluster(cluster_name, nodes, ipformat=None, start=True):
global CCM_CLUSTER
if USE_CASS_EXTERNAL:
if CCM_CLUSTER:
log.debug("Using external ccm cluster %s", CCM_CLUSTER.name)
else:
log.debug("Using unnamed external cluster")
return
if is_current_cluster(cluster_name, nodes):
log.debug("Using existing cluster %s", cluster_name)
return
if CCM_CLUSTER:
log.debug("Stopping cluster %s", CCM_CLUSTER.name)
CCM_CLUSTER.stop()
try:
try:
cluster = CCMClusterFactory.load(path, cluster_name)
log.debug("Found existing ccm %s cluster; clearing", cluster_name)
cluster.clear()
cluster.set_install_dir(**CCM_KWARGS)
except Exception:
log.debug("Creating new ccm %s cluster with %s", cluster_name, CCM_KWARGS)
cluster = CCMCluster(path, cluster_name, **CCM_KWARGS)
cluster.set_configuration_options({'start_native_transport': True})
if CASSANDRA_VERSION >= '2.2':
cluster.set_configuration_options({'enable_user_defined_functions': True})
common.switch_cluster(path, cluster_name)
cluster.populate(nodes, ipformat=ipformat)
jvm_args = []
# This will enable the Mirroring query handler which will echo our custom payload k,v pairs back
if PROTOCOL_VERSION >= 4:
jvm_args = [" -Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler"]
if start:
log.debug("Starting ccm %s cluster", cluster_name)
cluster.start(wait_for_binary_proto=True, wait_other_notice=True, jvm_args=jvm_args)
setup_keyspace(ipformat=ipformat)
CCM_CLUSTER = cluster
except Exception:
log.exception("Failed to start ccm cluster. Removing cluster.")
remove_cluster()
call(["pkill", "-9", "-f", ".ccm"])
raise
def run(self):
try:
cluster = Cluster(self.path,
self.name,
cassandra_dir=self.options.cassandra_dir,
cassandra_version=self.options.cassandra_version,
verbose=True)
except OSError as e:
cluster_dir = os.path.join(self.path, self.name)
import traceback
print >> sys.stderr, 'Cannot create cluster: %s\n%s' % (
str(e), traceback.format_exc())
exit(1)
if self.options.partitioner:
cluster.set_partitioner(self.options.partitioner)
if cluster.version() >= "1.2" and self.options.binary_protocol:
cluster.set_configuration_options({'start_native_transport': True})
if cluster.version() >= "1.2" and self.options.vnodes:
cluster.set_configuration_options({'num_tokens': 256})
if not self.options.no_switch:
common.switch_cluster(self.path, self.name)
print 'Current cluster is now: %s' % self.name
if self.nodes is not None:
try:
if self.options.debug_log:
cluster.set_log_level("DEBUG")
if self.options.trace_log:
cluster.set_log_level("TRACE")
cluster.populate(self.nodes,
use_vnodes=self.options.vnodes,
ipprefix=self.options.ipprefix)
if self.options.start_nodes:
cluster.start(
verbose=self.options.debug,
wait_for_binary_proto=self.options.binary_protocol)
except common.ArgumentError as e:
print >> sys.stderr, str(e)
exit(1)
def use_cluster(cluster_name, nodes, ipformat=None, start=True):
global CCM_CLUSTER
if USE_CASS_EXTERNAL:
if CCM_CLUSTER:
log.debug("Using external ccm cluster %s", CCM_CLUSTER.name)
else:
log.debug("Using unnamed external cluster")
return
if is_current_cluster(cluster_name, nodes):
log.debug("Using existing cluster %s", cluster_name)
return
if CCM_CLUSTER:
log.debug("Stopping cluster %s", CCM_CLUSTER.name)
CCM_CLUSTER.stop()
try:
try:
cluster = CCMClusterFactory.load(path, cluster_name)
log.debug("Found existing ccm %s cluster; clearing", cluster_name)
cluster.clear()
cluster.set_install_dir(**CCM_KWARGS)
except Exception:
log.debug("Creating new ccm %s cluster with %s", cluster_name, CCM_KWARGS)
cluster = CCMCluster(path, cluster_name, **CCM_KWARGS)
cluster.set_configuration_options({'start_native_transport': True})
if CASSANDRA_VERSION >= '2.2':
cluster.set_configuration_options({'enable_user_defined_functions': True})
common.switch_cluster(path, cluster_name)
cluster.populate(nodes, ipformat=ipformat)
if start:
log.debug("Starting ccm %s cluster", cluster_name)
cluster.start(wait_for_binary_proto=True, wait_other_notice=True)
setup_keyspace(ipformat=ipformat)
CCM_CLUSTER = cluster
except Exception:
log.exception("Failed to start ccm cluster. Removing cluster.")
remove_cluster()
call(["pkill", "-9", "-f", ".ccm"])
raise
def setup_package():
try:
try:
cluster = CCMCluster.load(path, CLUSTER_NAME)
log.debug("Found existing ccm test cluster, clearing")
cluster.clear()
except Exception:
log.debug("Creating new ccm test cluster")
cluster = CCMCluster(path, CLUSTER_NAME, cassandra_version='1.2.6')
cluster.set_configuration_options({'start_native_transport': True})
common.switch_cluster(path, CLUSTER_NAME)
cluster.populate(3)
log.debug("Starting ccm test cluster")
cluster.start(wait_for_binary_proto=True)
except Exception:
log.exception("Failed to start ccm cluster:")
raise
global CCM_CLUSTER
CCM_CLUSTER = cluster
def setup_package():
try:
try:
cluster = CCMCluster.load(path, CLUSTER_NAME)
log.debug("Found existing ccm test cluster, clearing")
cluster.clear()
except Exception:
log.debug("Creating new ccm test cluster")
cluster = CCMCluster(path, CLUSTER_NAME, cassandra_version='1.2.6')
cluster.set_configuration_options({'start_native_transport': True})
common.switch_cluster(path, CLUSTER_NAME)
cluster.populate(3)
log.debug("Starting ccm test cluster")
cluster.start(wait_for_binary_proto=True)
except Exception:
log.exception("Failed to start ccm cluster:")
raise
global CCM_CLUSTER
CCM_CLUSTER = cluster
def _start_cluster(self):
"""Docstring."""
try:
cluster = CCMClusterFactory.load(self.CLUSTER_PATH, self.CLUSTER_NAME)
logging.debug(
"Found existing ccm {} cluster; clearing".format(self.CLUSTER_NAME)
)
cluster.start(wait_for_binary_proto=True, wait_other_notice=True)
self.CCM_CLUSTER = cluster
except Exception:
logging.debug(
"Creating new ccm cluster {} with {}",
self.CLUSTER_NAME,
self.CLUSTER_KWARGS,
)
cluster = CCMCluster(
self.CLUSTER_PATH, self.CLUSTER_NAME, **self.CLUSTER_KWARGS
)
cluster.set_configuration_options({"start_native_transport": True})
common.switch_cluster(self.CLUSTER_PATH, self.CLUSTER_NAME)
cluster.populate(self.CLUSTER_NODE_COUNT, ipformat=None)
cluster.start(wait_for_binary_proto=True, wait_other_notice=True)
self.CCM_CLUSTER = cluster
def run(self):
try:
cluster = Cluster(self.path, self.name, cassandra_dir=self.options.cassandra_dir, cassandra_version=self.options.cassandra_version, verbose=True)
except OSError as e:
cluster_dir = os.path.join(self.path, self.name)
print >> sys.stderr, 'Cannot create cluster: %s' % str(e)
exit(1)
if self.options.partitioner:
cluster.set_partitioner(self.options.partitioner)
if not self.options.no_switch:
common.switch_cluster(self.path, self.name)
print 'Current cluster is now: %s' % self.name
if self.nodes is not None:
try:
cluster.populate(self.nodes)
if self.options.start_nodes:
cluster.start()
except common.ArgumentError as e:
print >> sys.stderr, str(e)
exit(1)
def setup_package():
version = os.getenv("CASSANDRA_VERSION", DEFAULT_CASSANDRA_VERSION)
try:
try:
cluster = CCMCluster.load(path, CLUSTER_NAME)
log.debug("Found existing ccm test cluster, clearing")
cluster.clear()
cluster.set_cassandra_dir(cassandra_version=version)
except Exception:
log.debug("Creating new ccm test cluster with version %s", version)
cluster = CCMCluster(path, CLUSTER_NAME, cassandra_version=version)
cluster.set_configuration_options({'start_native_transport': True})
common.switch_cluster(path, CLUSTER_NAME)
cluster.populate(3)
log.debug("Starting ccm test cluster")
cluster.start(wait_for_binary_proto=True)
except Exception:
log.exception("Failed to start ccm cluster:")
raise
global CCM_CLUSTER
CCM_CLUSTER = cluster
setup_test_keyspace()
def run(self):
common.switch_cluster(self.path, self.name)
def run(self):
common.switch_cluster(self.path, self.name)
def use_cluster(cluster_name, nodes, ipformat=None, start=True, workloads=None, set_keyspace=True, ccm_options=None,
configuration_options={}, dse_options={}, use_single_interface=USE_SINGLE_INTERFACE):
dse_cluster = True if DSE_VERSION else False
if not workloads:
workloads = []
if ccm_options is None and DSE_VERSION:
ccm_options = {"version": CCM_VERSION}
elif ccm_options is None:
ccm_options = CCM_KWARGS.copy()
cassandra_version = ccm_options.get('version', CCM_VERSION)
dse_version = ccm_options.get('version', DSE_VERSION)
global CCM_CLUSTER
if USE_CASS_EXTERNAL:
if CCM_CLUSTER:
log.debug("Using external CCM cluster {0}".format(CCM_CLUSTER.name))
else:
ccm_path = os.getenv("CCM_PATH", None)
ccm_name = os.getenv("CCM_NAME", None)
if ccm_path and ccm_name:
CCM_CLUSTER = CCMClusterFactory.load(ccm_path, ccm_name)
log.debug("Using external CCM cluster {0}".format(CCM_CLUSTER.name))
else:
log.debug("Using unnamed external cluster")
if set_keyspace and start:
setup_keyspace(ipformat=ipformat, wait=False)
return
if is_current_cluster(cluster_name, nodes, workloads):
log.debug("Using existing cluster, matching topology: {0}".format(cluster_name))
else:
if CCM_CLUSTER:
log.debug("Stopping existing cluster, topology mismatch: {0}".format(CCM_CLUSTER.name))
CCM_CLUSTER.stop()
try:
CCM_CLUSTER = CCMClusterFactory.load(path, cluster_name)
log.debug("Found existing CCM cluster, {0}; clearing.".format(cluster_name))
CCM_CLUSTER.clear()
CCM_CLUSTER.set_install_dir(**ccm_options)
CCM_CLUSTER.set_configuration_options(configuration_options)
CCM_CLUSTER.set_dse_configuration_options(dse_options)
except Exception:
ex_type, ex, tb = sys.exc_info()
log.warning("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb)))
del tb
ccm_options.update(cmd_line_args_to_dict('CCM_ARGS'))
log.debug("Creating new CCM cluster, {0}, with args {1}".format(cluster_name, ccm_options))
# Make sure we cleanup old cluster dir if it exists
cluster_path = os.path.join(path, cluster_name)
if os.path.exists(cluster_path):
shutil.rmtree(cluster_path)
if dse_cluster:
CCM_CLUSTER = DseCluster(path, cluster_name, **ccm_options)
CCM_CLUSTER.set_configuration_options({'start_native_transport': True})
CCM_CLUSTER.set_configuration_options({'batch_size_warn_threshold_in_kb': 5})
if Version(dse_version) >= Version('5.0'):
CCM_CLUSTER.set_configuration_options({'enable_user_defined_functions': True})
CCM_CLUSTER.set_configuration_options({'enable_scripted_user_defined_functions': True})
if Version(dse_version) >= Version('5.1'):
# For Inet4Address
CCM_CLUSTER.set_dse_configuration_options({
'graph': {
'gremlin_server': {
'scriptEngines': {
'gremlin-groovy': {
'config': {
'sandbox_rules': {
'whitelist_packages': ['java.net']
}
}
}
}
}
}
})
if 'spark' in workloads:
if Version(dse_version) >= Version('6.8'):
config_options = {
"resource_manager_options": {
"worker_options": {
"cores_total": 0.1,
"memory_total": "64M"
}
}
}
else:
config_options = {"initial_spark_worker_resources": 0.1}
if Version(dse_version) >= Version('6.7'):
log.debug("Disabling AlwaysON SQL for a DSE 6.7 Cluster")
config_options['alwayson_sql_options'] = {'enabled': False}
CCM_CLUSTER.set_dse_configuration_options(config_options)
common.switch_cluster(path, cluster_name)
CCM_CLUSTER.set_configuration_options(configuration_options)
CCM_CLUSTER.populate(nodes, ipformat=ipformat)
CCM_CLUSTER.set_dse_configuration_options(dse_options)
else:
CCM_CLUSTER = CCMCluster(path, cluster_name, **ccm_options)
CCM_CLUSTER.set_configuration_options({'start_native_transport': True})
if Version(cassandra_version) >= Version('2.2'):
CCM_CLUSTER.set_configuration_options({'enable_user_defined_functions': True})
if Version(cassandra_version) >= Version('3.0'):
CCM_CLUSTER.set_configuration_options({'enable_scripted_user_defined_functions': True})
if Version(cassandra_version) >= Version('4.0-a'):
CCM_CLUSTER.set_configuration_options({
'enable_materialized_views': True,
'enable_sasi_indexes': True,
'enable_transient_replication': True,
})
common.switch_cluster(path, cluster_name)
CCM_CLUSTER.set_configuration_options(configuration_options)
CCM_CLUSTER.populate(nodes, ipformat=ipformat, use_single_interface=use_single_interface)
try:
jvm_args = []
# This will enable the Mirroring query handler which will echo our custom payload k,v pairs back
if 'graph' in workloads:
jvm_args += ['-Xms1500M', '-Xmx1500M']
else:
if PROTOCOL_VERSION >= 4:
jvm_args = [" -Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler"]
if len(workloads) > 0:
for node in CCM_CLUSTER.nodes.values():
node.set_workloads(workloads)
if start:
log.debug("Starting CCM cluster: {0}".format(cluster_name))
CCM_CLUSTER.start(jvm_args=jvm_args, wait_for_binary_proto=True)
# Added to wait for slow nodes to start up
log.debug("Cluster started waiting for binary ports")
for node in CCM_CLUSTER.nodes.values():
wait_for_node_socket(node, 300)
log.debug("Binary ports are open")
if set_keyspace:
setup_keyspace(ipformat=ipformat)
except Exception:
log.exception("Failed to start CCM cluster; removing cluster.")
if os.name == "nt":
if CCM_CLUSTER:
for node in six.itervalues(CCM_CLUSTER.nodes):
os.system("taskkill /F /PID " + str(node.pid))
else:
call(["pkill", "-9", "-f", ".ccm"])
remove_cluster()
raise
return CCM_CLUSTER
def setup_class(cls):
"""
Class-level fixture. Called once for all tests.
We create a cluster object here and save it in the provided class instance
Create 2 regular users and a new superuser.
Enable authentication by setting 'authenticator': 'PasswordAuthenticator'.
If invoked from authorization class enable authorization by setting 'authorizer': 'CassandraAuthorizer'.
"""
try:
ccm_cluster = CCMCluster.load(tests.integration.path,
AUTH_CLUSTER_NAME)
log.debug(
"Found existing ccm test authentication cluster, removing")
ccm_cluster.remove()
except Exception:
log.debug("Can not load cluster %s ....." % AUTH_CLUSTER_NAME)
log.debug("Creating new ccm test authentication cluster")
if tests.integration.CASSANDRA_DIR:
ccm_cluster = CCMCluster(
tests.integration.path,
AUTH_CLUSTER_NAME,
cassandra_dir=tests.integration.CASSANDRA_DIR)
else:
ccm_cluster = CCMCluster(
tests.integration.path,
AUTH_CLUSTER_NAME,
cassandra_version=tests.integration.CASSANDRA_VERSION)
ccm_cluster.set_configuration_options({'start_native_transport': True})
ccm_cluster.set_configuration_options(
{'authenticator': 'PasswordAuthenticator'})
#
# This method is called either with AuthenticationTests class or with AuthorizedAuthenticationTests class.
# In the second case we enable CassandraAuthorizer
#
if cls.__name__ == 'AuthorizedAuthenticationTests':
print "Running tests with Cassandra Authorizer Enabled"
log.info("Running tests with Cassandra Authorizer Enabled")
ccm_cluster.set_configuration_options(
{'authorizer': 'CassandraAuthorizer'})
else:
print "Running tests with Cassandra Authorizer Disabled"
log.info("Running tests with Cassandra Authorizer Disabled")
common.switch_cluster(tests.integration.path, AUTH_CLUSTER_NAME)
ccm_cluster.populate(1)
log.debug("Starting ccm test authentication cluster")
ccm_cluster.start(wait_for_binary_proto=True)
if not wait_for_cassandra() or not try_connecting(
'cassandra', 'cassandra'):
log.exception("Can not talk to cassandra")
raise Exception('Can not talk to cassandra')
log.debug("Switched to AUTH_CLUSTER_NAME cluster")
cls.ccm_cluster = ccm_cluster
cls.root_cluster = cls.cluster_as('cassandra', 'cassandra')
cls.root_session = cls.root_cluster.connect()
cls.create_user(cls.root_cluster, cls.test_user, cls.password)
cls.create_user(cls.root_cluster, cls.test_other_user, cls.password)
cls.create_user(cls.root_cluster, cls.suser, cls.password, su=True)
def use_cluster(cluster_name, nodes, ipformat=None, start=True, workloads=[], set_keyspace=True, ccm_options=None,
configuration_options={}):
set_default_cass_ip()
if ccm_options is None:
ccm_options = CCM_KWARGS.copy()
cassandra_version = ccm_options.get('version', CASSANDRA_VERSION)
if 'version' in ccm_options:
ccm_options['version'] = ccm_options['version'].base_version
global CCM_CLUSTER
if USE_CASS_EXTERNAL:
if CCM_CLUSTER:
log.debug("Using external CCM cluster {0}".format(CCM_CLUSTER.name))
else:
log.debug("Using unnamed external cluster")
if set_keyspace and start:
setup_keyspace(ipformat=ipformat, wait=False)
return
if is_current_cluster(cluster_name, nodes):
log.debug("Using existing cluster, matching topology: {0}".format(cluster_name))
else:
if CCM_CLUSTER:
log.debug("Stopping existing cluster, topology mismatch: {0}".format(CCM_CLUSTER.name))
CCM_CLUSTER.stop()
try:
CCM_CLUSTER = CCMClusterFactory.load(path, cluster_name)
log.debug("Found existing CCM cluster, {0}; clearing.".format(cluster_name))
CCM_CLUSTER.clear()
CCM_CLUSTER.set_install_dir(**ccm_options)
CCM_CLUSTER.set_configuration_options(configuration_options)
except Exception:
ex_type, ex, tb = sys.exc_info()
log.warning("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb)))
del tb
log.debug("Creating new CCM cluster, {0}, with args {1}".format(cluster_name, ccm_options))
CCM_CLUSTER = CCMCluster(path, cluster_name, **ccm_options)
CCM_CLUSTER.set_configuration_options({'start_native_transport': True})
if cassandra_version >= Version('2.2'):
CCM_CLUSTER.set_configuration_options({'enable_user_defined_functions': True})
if cassandra_version >= Version('3.0'):
CCM_CLUSTER.set_configuration_options({'enable_scripted_user_defined_functions': True})
common.switch_cluster(path, cluster_name)
CCM_CLUSTER.set_configuration_options(configuration_options)
CCM_CLUSTER.populate(nodes, ipformat=ipformat)
try:
jvm_args = []
# This will enable the Mirroring query handler which will echo our custom payload k,v pairs back
if 'graph' not in workloads:
if PROTOCOL_VERSION >= 4:
jvm_args = [" -Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler"]
if(len(workloads) > 0):
for node in CCM_CLUSTER.nodes.values():
node.set_workloads(workloads)
if start:
log.debug("Starting CCM cluster: {0}".format(cluster_name))
CCM_CLUSTER.start(wait_for_binary_proto=True, wait_other_notice=True, jvm_args=jvm_args)
# Added to wait for slow nodes to start up
for node in CCM_CLUSTER.nodes.values():
wait_for_node_socket(node, 120)
if set_keyspace:
setup_keyspace(ipformat=ipformat)
except Exception:
log.exception("Failed to start CCM cluster; removing cluster.")
if os.name == "nt":
if CCM_CLUSTER:
for node in six.itervalues(CCM_CLUSTER.nodes):
os.system("taskkill /F /PID " + str(node.pid))
else:
call(["pkill", "-9", "-f", ".ccm"])
remove_cluster()
raise
return CCM_CLUSTER
def use_cluster(cluster_name, nodes, ipformat=None, start=True, workloads=[], set_keyspace=True, ccm_options=None,
configuration_options={}, dse_cluster=False, dse_options={},
dse_version=None):
if (dse_version and not dse_cluster):
raise ValueError('specified dse_version {} but not dse_cluster'.format(dse_version))
set_default_cass_ip()
if ccm_options is None and dse_cluster:
ccm_options = {"version": dse_version or DSE_VERSION}
elif ccm_options is None:
ccm_options = CCM_KWARGS.copy()
cassandra_version = ccm_options.get('version', CCM_VERSION)
dse_version = ccm_options.get('version', DSE_VERSION)
if 'version' in ccm_options:
ccm_options['version'] = ccm_options['version'].base_version
global CCM_CLUSTER
if USE_CASS_EXTERNAL:
if CCM_CLUSTER:
log.debug("Using external CCM cluster {0}".format(CCM_CLUSTER.name))
else:
log.debug("Using unnamed external cluster")
if set_keyspace and start:
setup_keyspace(ipformat=ipformat, wait=False)
return
if is_current_cluster(cluster_name, nodes):
log.debug("Using existing cluster, matching topology: {0}".format(cluster_name))
else:
if CCM_CLUSTER:
log.debug("Stopping existing cluster, topology mismatch: {0}".format(CCM_CLUSTER.name))
CCM_CLUSTER.stop()
try:
CCM_CLUSTER = CCMClusterFactory.load(path, cluster_name)
log.debug("Found existing CCM cluster, {0}; clearing.".format(cluster_name))
CCM_CLUSTER.clear()
CCM_CLUSTER.set_install_dir(**ccm_options)
CCM_CLUSTER.set_configuration_options(configuration_options)
except Exception:
ex_type, ex, tb = sys.exc_info()
log.warning("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb)))
del tb
ccm_options.update(cmd_line_args_to_dict('CCM_ARGS'))
log.debug("Creating new CCM cluster, {0}, with args {1}".format(cluster_name, ccm_options))
# Make sure we cleanup old cluster dir if it exists
cluster_path = os.path.join(path, cluster_name)
if os.path.exists(cluster_path):
shutil.rmtree(cluster_path)
if dse_cluster:
CCM_CLUSTER = DseCluster(path, cluster_name, **ccm_options)
CCM_CLUSTER.set_configuration_options({'start_native_transport': True})
CCM_CLUSTER.set_configuration_options({'batch_size_warn_threshold_in_kb': 5})
if dse_version >= Version('5.0'):
CCM_CLUSTER.set_configuration_options({'enable_user_defined_functions': True})
CCM_CLUSTER.set_configuration_options({'enable_scripted_user_defined_functions': True})
if 'spark' in workloads:
config_options = {"initial_spark_worker_resources": 0.1}
CCM_CLUSTER.set_dse_configuration_options(config_options)
common.switch_cluster(path, cluster_name)
CCM_CLUSTER.set_configuration_options(configuration_options)
CCM_CLUSTER.populate(nodes, ipformat=ipformat)
CCM_CLUSTER.set_dse_configuration_options(dse_options)
else:
CCM_CLUSTER = CCMCluster(path, cluster_name, **ccm_options)
CCM_CLUSTER.set_configuration_options({'start_native_transport': True})
if cassandra_version >= Version('2.2'):
CCM_CLUSTER.set_configuration_options({'enable_user_defined_functions': True})
if cassandra_version >= Version('3.0'):
CCM_CLUSTER.set_configuration_options({'enable_scripted_user_defined_functions': True})
common.switch_cluster(path, cluster_name)
CCM_CLUSTER.set_configuration_options(configuration_options)
CCM_CLUSTER.populate(nodes, ipformat=ipformat)
try:
jvm_args = []
# This will enable the Mirroring query handler which will echo our custom payload k,v pairs back
if 'graph' not in workloads:
if PROTOCOL_VERSION >= 4:
jvm_args = [" -Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler"]
if(len(workloads) > 0):
for node in CCM_CLUSTER.nodes.values():
node.set_workloads(workloads)
if start:
log.debug("Starting CCM cluster: {0}".format(cluster_name))
CCM_CLUSTER.start(wait_for_binary_proto=True, wait_other_notice=True, jvm_args=jvm_args)
# Added to wait for slow nodes to start up
for node in CCM_CLUSTER.nodes.values():
wait_for_node_socket(node, 120)
if set_keyspace:
setup_keyspace(ipformat=ipformat)
except Exception:
log.exception("Failed to start CCM cluster; removing cluster.")
if os.name == "nt":
if CCM_CLUSTER:
for node in six.itervalues(CCM_CLUSTER.nodes):
os.system("taskkill /F /PID " + str(node.pid))
else:
call(["pkill", "-9", "-f", ".ccm"])
remove_cluster()
raise
return CCM_CLUSTER
def use_cluster(cluster_name, nodes, ipformat=None, start=True, workloads=[]):
set_default_dse_ip()
global CCM_CLUSTER
if USE_CASS_EXTERNAL:
if CCM_CLUSTER:
log.debug("Using external CCM cluster {0}".format(
CCM_CLUSTER.name))
else:
log.debug("Using unnamed external cluster")
setup_keyspace(ipformat=ipformat, wait=False)
return
if is_current_cluster(cluster_name, nodes, workloads):
log.debug("Using existing cluster, matching topology: {0}".format(
cluster_name))
else:
if CCM_CLUSTER:
log.debug(
"Stopping existing cluster, topology mismatch: {0}".format(
CCM_CLUSTER.name))
CCM_CLUSTER.stop()
try:
CCM_CLUSTER = CCMClusterFactory.load(path, cluster_name)
log.debug("Found existing CCM cluster, {0}; clearing.".format(
cluster_name))
CCM_CLUSTER.clear()
CCM_CLUSTER.set_install_dir(**CCM_KWARGS)
except Exception:
ex_type, ex, tb = sys.exc_info()
log.warn("{0}: {1} Backtrace: {2}".format(
ex_type.__name__, ex, traceback.extract_tb(tb)))
del tb
log.debug("Creating new CCM cluster, {0}, with args {1}".format(
cluster_name, CCM_KWARGS))
if DSE_VERSION:
log.error("creating dse cluster")
CCM_CLUSTER = DseCluster(path, cluster_name, **CCM_KWARGS)
else:
CCM_CLUSTER = CCMCluster(path, cluster_name, **CCM_KWARGS)
CCM_CLUSTER.set_configuration_options(
{'start_native_transport': True})
CCM_CLUSTER.set_configuration_options(
{'batch_size_warn_threshold_in_kb': 5})
if CASSANDRA_VERSION >= '2.2':
CCM_CLUSTER.set_configuration_options(
{'enable_user_defined_functions': True})
if CASSANDRA_VERSION >= '3.0':
CCM_CLUSTER.set_configuration_options(
{'enable_scripted_user_defined_functions': True})
if 'spark' in workloads:
config_options = {"initial_spark_worker_resources": 0.1}
CCM_CLUSTER.set_dse_configuration_options(config_options)
common.switch_cluster(path, cluster_name)
CCM_CLUSTER.populate(nodes, ipformat=ipformat)
try:
jvm_args = []
# This will enable the Mirroring query handler which will echo our custom payload k,v pairs back
if 'graph' not in workloads:
if PROTOCOL_VERSION >= 4:
jvm_args = [
" -Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler"
]
if (len(workloads) > 0):
for node in CCM_CLUSTER.nodes.values():
node.set_workloads(workloads)
if start:
log.debug("Starting CCM cluster: {0}".format(cluster_name))
CCM_CLUSTER.start(no_wait=True, jvm_args=jvm_args)
# Added to wait for slow nodes to start up
log.debug("Cluster started waiting for binary ports")
for node in CCM_CLUSTER.nodes.values():
wait_for_node_socket(node, 120)
log.debug("Binary port are open")
setup_keyspace(ipformat=ipformat)
except Exception:
log.exception("Failed to start CCM cluster; removing cluster.")
if os.name == "nt":
if CCM_CLUSTER:
for node in CCM_CLUSTER.nodes.itervalues():
os.system("taskkill /F /PID " + str(node.pid))
else:
call(["pkill", "-9", "-f", ".ccm"])
remove_cluster()
raise
def use_cluster(cluster_name,
nodes,
ipformat=None,
start=True,
workloads=[],
set_keyspace=True,
ccm_options=None,
configuration_options={},
dse_cluster=False,
dse_options={},
dse_version=None):
if (dse_version and not dse_cluster):
raise ValueError(
'specified dse_version {} but not dse_cluster'.format(dse_version))
set_default_cass_ip()
if ccm_options is None and dse_cluster:
ccm_options = {"version": dse_version or DSE_VERSION}
elif ccm_options is None:
ccm_options = CCM_KWARGS.copy()
cassandra_version = ccm_options.get('version', CCM_VERSION)
dse_version = ccm_options.get('version', DSE_VERSION)
if 'version' in ccm_options:
ccm_options['version'] = ccm_options['version'].base_version
global CCM_CLUSTER
if USE_CASS_EXTERNAL:
if CCM_CLUSTER:
log.debug("Using external CCM cluster {0}".format(
CCM_CLUSTER.name))
else:
log.debug("Using unnamed external cluster")
if set_keyspace and start:
setup_keyspace(ipformat=ipformat, wait=False)
return
if is_current_cluster(cluster_name, nodes):
log.debug("Using existing cluster, matching topology: {0}".format(
cluster_name))
else:
if CCM_CLUSTER:
log.debug(
"Stopping existing cluster, topology mismatch: {0}".format(
CCM_CLUSTER.name))
CCM_CLUSTER.stop()
try:
CCM_CLUSTER = CCMClusterFactory.load(path, cluster_name)
log.debug("Found existing CCM cluster, {0}; clearing.".format(
cluster_name))
CCM_CLUSTER.clear()
CCM_CLUSTER.set_install_dir(**ccm_options)
CCM_CLUSTER.set_configuration_options(configuration_options)
except Exception:
ex_type, ex, tb = sys.exc_info()
log.warning("{0}: {1} Backtrace: {2}".format(
ex_type.__name__, ex, traceback.extract_tb(tb)))
del tb
ccm_options.update(cmd_line_args_to_dict('CCM_ARGS'))
log.debug("Creating new CCM cluster, {0}, with args {1}".format(
cluster_name, ccm_options))
if dse_cluster:
CCM_CLUSTER = DseCluster(path, cluster_name, **ccm_options)
CCM_CLUSTER.set_configuration_options(
{'start_native_transport': True})
CCM_CLUSTER.set_configuration_options(
{'batch_size_warn_threshold_in_kb': 5})
if dse_version >= Version('5.0'):
CCM_CLUSTER.set_configuration_options(
{'enable_user_defined_functions': True})
CCM_CLUSTER.set_configuration_options(
{'enable_scripted_user_defined_functions': True})
if 'spark' in workloads:
config_options = {"initial_spark_worker_resources": 0.1}
CCM_CLUSTER.set_dse_configuration_options(config_options)
common.switch_cluster(path, cluster_name)
CCM_CLUSTER.set_configuration_options(configuration_options)
CCM_CLUSTER.populate(nodes, ipformat=ipformat)
CCM_CLUSTER.set_dse_configuration_options(dse_options)
else:
CCM_CLUSTER = CCMCluster(path, cluster_name, **ccm_options)
CCM_CLUSTER.set_configuration_options(
{'start_native_transport': True})
if cassandra_version >= Version('2.2'):
CCM_CLUSTER.set_configuration_options(
{'enable_user_defined_functions': True})
if cassandra_version >= Version('3.0'):
CCM_CLUSTER.set_configuration_options(
{'enable_scripted_user_defined_functions': True})
common.switch_cluster(path, cluster_name)
CCM_CLUSTER.set_configuration_options(configuration_options)
CCM_CLUSTER.populate(nodes, ipformat=ipformat)
try:
jvm_args = []
# This will enable the Mirroring query handler which will echo our custom payload k,v pairs back
if 'graph' not in workloads:
if PROTOCOL_VERSION >= 4:
jvm_args = [
" -Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler"
]
if (len(workloads) > 0):
for node in CCM_CLUSTER.nodes.values():
node.set_workloads(workloads)
if start:
log.debug("Starting CCM cluster: {0}".format(cluster_name))
CCM_CLUSTER.start(wait_for_binary_proto=True,
wait_other_notice=True,
jvm_args=jvm_args)
# Added to wait for slow nodes to start up
for node in CCM_CLUSTER.nodes.values():
wait_for_node_socket(node, 120)
if set_keyspace:
setup_keyspace(ipformat=ipformat)
except Exception:
log.exception("Failed to start CCM cluster; removing cluster.")
if os.name == "nt":
if CCM_CLUSTER:
for node in six.itervalues(CCM_CLUSTER.nodes):
os.system("taskkill /F /PID " + str(node.pid))
else:
call(["pkill", "-9", "-f", ".ccm"])
remove_cluster()
raise
return CCM_CLUSTER
