def _get_cluster(self, name='test'):
if self._preserve_cluster and hasattr(self, 'cluster'):
return self.cluster
self.test_path = tempfile.mkdtemp(prefix='dtest-')
# ccm on cygwin needs absolute path to directory - it crosses from cygwin space into
# regular Windows space on wmic calls which will otherwise break pathing
if sys.platform == "cygwin":
self.test_path = subprocess.Popen(["cygpath", "-m", self.test_path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()[0].rstrip()
debug("cluster ccm directory: " + self.test_path)
version = os.environ.get('CASSANDRA_VERSION')
cdir = CASSANDRA_DIR
if version:
cluster = Cluster(self.test_path, name, cassandra_version=version)
else:
cluster = Cluster(self.test_path, name, cassandra_dir=cdir)
if DISABLE_VNODES:
cluster.set_configuration_options(values={'num_tokens': None})
else:
cluster.set_configuration_options(values={'initial_token': None, 'num_tokens': NUM_TOKENS})
if OFFHEAP_MEMTABLES:
cluster.set_configuration_options(values={'memtable_allocation_type': 'offheap_objects'})
cluster.set_datadir_count(DATADIR_COUNT)
return cluster
def clear_and_use_multidc(dc_list):
teardown_package()
try:
try:
cluster = CCMCluster.load(path, MULTIDC_CLUSTER_NAME)
log.debug("Found existing ccm test multi-dc cluster, clearing")
cluster.clear()
except Exception:
log.debug("Creating new ccm test multi-dc cluster")
if CASSANDRA_DIR:
cluster = CCMCluster(path, MULTIDC_CLUSTER_NAME, cassandra_dir=CASSANDRA_DIR)
else:
cluster = CCMCluster(path, MULTIDC_CLUSTER_NAME, cassandra_version=CASSANDRA_VERSION)
cluster.set_configuration_options({'start_native_transport': True})
common.switch_cluster(path, MULTIDC_CLUSTER_NAME)
cluster.populate(dc_list)
log.debug("Starting ccm test cluster")
cluster.start(wait_for_binary_proto=True, wait_other_notice=True)
except Exception:
log.exception("Failed to start ccm cluster:")
raise
global CCM_CLUSTER
CCM_CLUSTER = cluster
setup_test_keyspace()
log.debug("Switched to multidc cluster")
def setup_module(cls):
validate_ccm_viable()
validate_host_viable()
cls.ccm_cluster = object()
teardown_package()
try:
try:
ccm_cluster = CCMCluster.load(path, IPV6_CLUSTER_NAME)
log.debug("Found existing ccm test ipv6 cluster, clearing")
ccm_cluster.clear()
except Exception:
log.debug("Creating new ccm test ipv6 cluster")
if CASSANDRA_DIR:
ccm_cluster = CCMCluster(path, IPV6_CLUSTER_NAME, cassandra_dir=CASSANDRA_DIR)
else:
ccm_cluster = CCMCluster(path, IPV6_CLUSTER_NAME, cassandra_version=CASSANDRA_VERSION)
ccm_cluster.set_configuration_options({'start_native_transport': True})
common.switch_cluster(path, IPV6_CLUSTER_NAME)
ccm_cluster.populate(1, ipformat='::%d')
log.debug("Starting ccm test cluster")
ccm_cluster.start(wait_for_binary_proto=True)
except Exception:
log.exception("Failed to start ccm cluster:")
raise
log.debug("Switched to ipv6 cluster")
cls.ccm_cluster = ccm_cluster
def setup_package():
print_('Using Cassandra version: %s' % CASSANDRA_VERSION)
try:
try:
cluster = CCMCluster.load(path, CLUSTER_NAME)
log.debug("Found existing ccm test cluster, clearing")
cluster.clear()
if CASSANDRA_DIR:
cluster.set_cassandra_dir(cassandra_dir=CASSANDRA_DIR)
else:
cluster.set_cassandra_dir(cassandra_version=CASSANDRA_VERSION)
except Exception:
if CASSANDRA_DIR:
log.debug("Creating new ccm test cluster with cassandra dir %s", CASSANDRA_DIR)
cluster = CCMCluster(path, CLUSTER_NAME, cassandra_dir=CASSANDRA_DIR)
else:
log.debug("Creating new ccm test cluster with version %s", CASSANDRA_VERSION)
cluster = CCMCluster(path, CLUSTER_NAME, cassandra_version=CASSANDRA_VERSION)
cluster.set_configuration_options({'start_native_transport': True})
common.switch_cluster(path, CLUSTER_NAME)
cluster.populate(3)
log.debug("Starting ccm test cluster")
cluster.start(wait_for_binary_proto=True, wait_other_notice=True)
except Exception:
log.exception("Failed to start ccm cluster:")
raise
global CCM_CLUSTER
CCM_CLUSTER = cluster
setup_test_keyspace()
def __get_cluster(self, name="test"):
self.test_path = tempfile.mkdtemp(prefix="dtest-")
try:
version = os.environ["CASSANDRA_VERSION"]
cluster = Cluster(self.test_path, name, cassandra_version=version)
except KeyError:
try:
cdir = os.environ["CASSANDRA_DIR"]
except KeyError:
cdir = DEFAULT_DIR
cluster = Cluster(self.test_path, name, cassandra_dir=cdir)
if ENABLE_VNODES:
cluster.set_configuration_options(values={"initial_token": None, "num_tokens": 256})
return cluster
def __get_cluster(self, name='test'):
self.test_path = tempfile.mkdtemp(prefix='dtest-')
try:
version = os.environ['CASSANDRA_VERSION']
cluster = Cluster(self.test_path, name, cassandra_version=version)
except KeyError:
try:
cdir = os.environ['CASSANDRA_DIR']
except KeyError:
cdir = DEFAULT_DIR
cluster = Cluster(self.test_path, name, cassandra_dir=cdir)
if ENABLE_VNODES:
cluster.set_configuration_options(values={'initial_token': None, 'num_tokens': 256})
return cluster
def use_cluster(cluster_name, nodes, ipformat=None, start=True):
global CCM_CLUSTER
if USE_CASS_EXTERNAL:
if CCM_CLUSTER:
log.debug("Using external CCM cluster {0}".format(CCM_CLUSTER.name))
else:
log.debug("Using unnamed external cluster")
return
if is_current_cluster(cluster_name, nodes):
log.debug("Using existing cluster, matching topology: {0}".format(cluster_name))
else:
if CCM_CLUSTER:
log.debug("Stopping existing cluster, topology mismatch: {0}".format(CCM_CLUSTER.name))
CCM_CLUSTER.stop()
try:
CCM_CLUSTER = CCMClusterFactory.load(path, cluster_name)
log.debug("Found existing CCM cluster, {0}; clearing.".format(cluster_name))
CCM_CLUSTER.clear()
CCM_CLUSTER.set_install_dir(**CCM_KWARGS)
except Exception:
ex_type, ex, tb = sys.exc_info()
log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb)))
del tb
log.debug("Creating new CCM cluster, {0}, with args {1}".format(cluster_name, CCM_KWARGS))
CCM_CLUSTER = CCMCluster(path, cluster_name, **CCM_KWARGS)
CCM_CLUSTER.set_configuration_options({'start_native_transport': True})
if CASSANDRA_VERSION >= '2.2':
CCM_CLUSTER.set_configuration_options({'enable_user_defined_functions': True})
common.switch_cluster(path, cluster_name)
CCM_CLUSTER.populate(nodes, ipformat=ipformat)
try:
jvm_args = []
# This will enable the Mirroring query handler which will echo our custom payload k,v pairs back
if PROTOCOL_VERSION >= 4:
jvm_args = [" -Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler"]
if start:
log.debug("Starting CCM cluster: {0}".format(cluster_name))
CCM_CLUSTER.start(wait_for_binary_proto=True, wait_other_notice=True, jvm_args=jvm_args)
setup_keyspace(ipformat=ipformat)
except Exception:
log.exception("Failed to start CCM cluster; removing cluster.")
if os.name == "nt":
if CCM_CLUSTER:
for node in CCM_CLUSTER.nodes.itervalues():
os.system("taskkill /F /PID " + str(node.pid))
else:
call(["pkill", "-9", "-f", ".ccm"])
remove_cluster()
raise
def setUp(self):
'''Create a cluster for cqlsh tests. Assumes that ccmtest.Tester's
teardown() method will safely stop and remove self.cluster.'''
self.cluster = Cluster(CLUSTER_PATH, "run_cqlsh",
cassandra_version='git:trunk')
self.cluster.populate(1).start(wait_for_binary_proto=True)
[self.node] = self.cluster.nodelist()
def replace_test(self):
cluster = self.cluster
tokens = Cluster.balanced_tokens(3)
cluster.populate(3, tokens=tokens).start()
[node1, node2, node3] = cluster.nodelist()
cursor = self.cql_connection(node1).cursor()
self.create_ks(cursor, 'ks', 3)
self.create_cf(cursor, 'cf')
for n in xrange(0, 10000):
insert_c1c2(cursor, n, "QUORUM")
cluster.flush()
node3.stop(wait_other_notice=True)
time.sleep(.5)
node4 = new_node(cluster, token=tokens[2])
node4.start(replace_token=tokens[2])
time.sleep(.5)
cluster.cleanup()
time.sleep(.5)
for n in xrange(0, 10000):
query_c1c2(cursor, n, "QUORUM")
sizes = [ node.data_size() for node in cluster.nodelist() if node.is_running()]
assert_almost_equal(*sizes)
def setUp(self):
# cleaning up if a previous execution didn't trigger tearDown (which
# can happen if it is interrupted by KeyboardInterrupt)
# TODO: move that part to a generic fixture
if os.path.exists(LAST_TEST_DIR):
with open(LAST_TEST_DIR) as f:
self.test_path = f.readline().strip("\n")
name = f.readline()
try:
self.cluster = Cluster.load(self.test_path, name)
# Avoid waiting too long for node to be marked down
self.__cleanup_cluster()
except IOError:
# after a restart, /tmp will be emptied so we'll get an IOError when loading the old cluster here
pass
self.cluster = self.__get_cluster()
# the failure detector can be quite slow in such tests with quick start/stop
self.cluster.set_configuration_options(values={"phi_convict_threshold": 5})
self.cluster.set_configuration_options(values={"rpc_timeout_in_ms": 15000})
with open(LAST_TEST_DIR, "w") as f:
f.write(self.test_path + "\n")
f.write(self.cluster.name)
if DEBUG:
self.cluster.set_log_level("DEBUG")
self.connections = []
self.runners = []
def test3(self):
self.cluster = Cluster(CLUSTER_PATH, "test3", cassandra_version='2.0.3')
self.cluster.populate(2)
self.cluster.start()
self.cluster.cleanup()
self.cluster.clear()
self.cluster.stop()
def use_cluster(cluster_name, nodes, ipformat=None, start=True):
if is_current_cluster(cluster_name, nodes):
log.debug("Using existing cluster %s", cluster_name)
return
global CCM_CLUSTER
if CCM_CLUSTER:
log.debug("Stopping cluster %s", CCM_CLUSTER.name)
CCM_CLUSTER.stop()
try:
try:
cluster = CCMClusterFactory.load(path, cluster_name)
log.debug("Found existing ccm %s cluster; clearing", cluster_name)
cluster.clear()
cluster.set_install_dir(**CCM_KWARGS)
except Exception:
log.debug("Creating new ccm %s cluster with %s", cluster_name, CCM_KWARGS)
cluster = CCMCluster(path, cluster_name, **CCM_KWARGS)
cluster.set_configuration_options({'start_native_transport': True})
common.switch_cluster(path, cluster_name)
cluster.populate(nodes, ipformat=ipformat)
if start:
log.debug("Starting ccm %s cluster", cluster_name)
cluster.start(wait_for_binary_proto=True, wait_other_notice=True)
setup_test_keyspace()
CCM_CLUSTER = cluster
except Exception:
log.exception("Failed to start ccm cluster:")
raise
def restart_test(self):
self.cluster = Cluster(CLUSTER_PATH, "restart", cassandra_version='2.0.9')
self.cluster.populate(3)
self.cluster.start()
self.cluster.stop()
self.cluster.start()
self.cluster.show(True)
def use_cluster(cluster_name, nodes, ipformat=None, start=True):
global CCM_CLUSTER
if USE_CASS_EXTERNAL:
if CCM_CLUSTER:
log.debug("Using external ccm cluster %s", CCM_CLUSTER.name)
else:
log.debug("Using unnamed external cluster")
return
if is_current_cluster(cluster_name, nodes):
log.debug("Using existing cluster %s", cluster_name)
return
if CCM_CLUSTER:
log.debug("Stopping cluster %s", CCM_CLUSTER.name)
CCM_CLUSTER.stop()
try:
try:
cluster = CCMClusterFactory.load(path, cluster_name)
log.debug("Found existing ccm %s cluster; clearing", cluster_name)
cluster.clear()
cluster.set_install_dir(**CCM_KWARGS)
except Exception:
log.debug("Creating new ccm %s cluster with %s", cluster_name, CCM_KWARGS)
cluster = CCMCluster(path, cluster_name, **CCM_KWARGS)
cluster.set_configuration_options({'start_native_transport': True})
if CASSANDRA_VERSION >= '2.2':
cluster.set_configuration_options({'enable_user_defined_functions': True})
common.switch_cluster(path, cluster_name)
cluster.populate(nodes, ipformat=ipformat)
jvm_args = []
# This will enable the Mirroring query handler which will echo our custom payload k,v pairs back
if PROTOCOL_VERSION >= 4:
jvm_args = [" -Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler"]
if start:
log.debug("Starting ccm %s cluster", cluster_name)
cluster.start(wait_for_binary_proto=True, wait_other_notice=True, jvm_args=jvm_args)
setup_keyspace(ipformat=ipformat)
CCM_CLUSTER = cluster
except Exception:
log.exception("Failed to start ccm cluster. Removing cluster.")
remove_cluster()
call(["pkill", "-9", "-f", ".ccm"])
raise
def _load_current_cluster(self):
name = common.current_cluster_name(self.path)
if name is None:
print_('No currently active cluster (use ccm cluster switch)')
exit(1)
try:
return Cluster.load(self.path, name)
except common.LoadError as e:
print_(str(e))
exit(1)
class IntegrationTest(TestCase):
def setUp(self):
self.test_path = tempfile.mkdtemp(prefix=self.name)
print
print "CCM using %s" % self.test_path
# ccm setup
self.session = None
shutil.rmtree(self.test_path)
os.makedirs(self.test_path)
self.cluster = Cluster(self.test_path, "test", cassandra_version="3.0.0")
# sstable tools setup
self.uberjar_location = glob.glob("%s/../../../target/sstable-*.jar" % os.path.dirname(os.path.realpath(__file__)))[0]
print "Using sstable build located at: %s" % self.uberjar_location
def cql_connection(self, node, keyspace=None, timeout=60):
if self.session:
return self.session
deadline = time.time() + timeout
while True:
try:
self.connection_cluster = PyCluster([node.network_interfaces['binary'][0]],
port=node.network_interfaces['binary'][1])
self.session = self.connection_cluster.connect()
if keyspace:
self.session.execute(("CREATE KEYSPACE IF NOT EXISTS %s WITH REPLICATION = " +
"{ 'class' : 'SimpleStrategy', 'replication_factor' : 1 };") % keyspace)
self.session.execute("USE %s" % keyspace)
return self.session
except NoHostAvailable:
if time.time() > deadline:
raise
else:
time.sleep(0.25)
def tearDown(self):
self.cluster.stop()
def simple_bootstrap_test(self):
cluster = self.cluster
if cluster.version() >= '1.2':
tokens = Cluster.balanced_tokens(2, 64)
else:
tokens = Cluster.balanced_tokens(2, 128)
keys = 10000
# Create a single node cluster
cluster.populate(1, tokens=[tokens[0]]).start()
node1 = cluster.nodes["node1"]
time.sleep(.5)
cursor = self.cql_connection(node1).cursor()
self.create_ks(cursor, 'ks', 1)
self.create_cf(cursor, 'cf', columns={ 'c1' : 'text', 'c2' : 'text' })
for n in xrange(0, keys):
insert_c1c2(cursor, n, "ONE")
node1.flush()
initial_size = node1.data_size()
# Reads inserted data all during the boostrap process. We shouldn't
# get any error
reader = self.go(lambda _: query_c1c2(cursor, random.randint(0, keys-1), "ONE"))
# Boostraping a new node
node2 = new_node(cluster, token=tokens[1])
node2.start()
time.sleep(.5)
reader.check()
node1.cleanup()
time.sleep(.5)
reader.check()
size1 = node1.data_size()
size2 = node2.data_size()
assert_almost_equal(size1, size2)
assert_almost_equal(initial_size, 2 * size1)
class TestRunCqlsh(ccmtest.Tester):
def setUp(self):
'''Create a cluster for cqlsh tests. Assumes that ccmtest.Tester's
teardown() method will safely stop and remove self.cluster.'''
self.cluster = Cluster(CLUSTER_PATH, "run_cqlsh",
cassandra_version='git:trunk')
self.cluster.populate(1).start(wait_for_binary_proto=True)
[self.node] = self.cluster.nodelist()
def run_cqlsh_printing(self, return_output, show_output):
'''Parameterized test. Runs run_cqlsh with options to print the output
and to return it as a string, or with these options combined, depending
on the values of the arguments.'''
# redirect run_cqlsh's stdout to a string buffer
old_stdout, sys.stdout = sys.stdout, StringIO()
rv = self.node.run_cqlsh('DESCRIBE keyspaces;',
return_output=return_output,
show_output=show_output)
# put stdout back where it belongs and get the built string value
sys.stdout, printed_output = old_stdout, sys.stdout.getvalue()
if return_output:
# we should see names of system keyspaces
self.assertIn('system', rv[0])
# stderr should be empty
self.assertEqual('', rv[1])
else:
# implicitly-returned None
self.assertEqual(rv, None)
if show_output:
self.assertIn('system', printed_output)
else:
# nothing should be printed if (not show_output)
self.assertEqual(printed_output, '')
if return_output and show_output:
self.assertEqual(printed_output, rv[0])
def simple_test(self, version='2.0.9'):
self.cluster = Cluster(CLUSTER_PATH, "simple", cassandra_version=version)
self.cluster.populate(3)
self.cluster.start()
node1, node2, node3 = self.cluster.nodelist()
if version < '2.1':
node1.stress()
else:
node1.stress(['write', 'n=1000000'])
self.cluster.flush()
def multi_dc_test(self):
self.cluster = Cluster(CLUSTER_PATH, "multi_dc", cassandra_version='2.0.9')
self.cluster.populate([1, 2])
self.cluster.start()
dcs = [node.data_center for node in self.cluster.nodelist()]
self.cluster.set_configuration_options(None, None)
self.cluster.stop()
self.cluster.start()
dcs_2 = [node.data_center for node in self.cluster.nodelist()]
self.assertListEqual(dcs, dcs_2)
def run(self):
if self.other_cluster:
# Remove the specified cluster:
cluster = Cluster.load(self.path, self.other_cluster)
cluster.remove()
# Remove CURRENT flag if the specified cluster is the current cluster:
if self.other_cluster == common.current_cluster_name(self.path):
os.remove(os.path.join(self.path, 'CURRENT'))
else:
# Remove the current cluster:
self.cluster.remove()
os.remove(os.path.join(self.path, 'CURRENT'))
def use_cluster(cluster_name, nodes, ipformat=None, start=True):
global CCM_CLUSTER
if USE_CASS_EXTERNAL:
if CCM_CLUSTER:
log.debug("Using external ccm cluster %s", CCM_CLUSTER.name)
else:
log.debug("Using unnamed external cluster")
return
if is_current_cluster(cluster_name, nodes):
log.debug("Using existing cluster %s", cluster_name)
return
if CCM_CLUSTER:
log.debug("Stopping cluster %s", CCM_CLUSTER.name)
CCM_CLUSTER.stop()
try:
try:
cluster = CCMClusterFactory.load(path, cluster_name)
log.debug("Found existing ccm %s cluster; clearing", cluster_name)
cluster.clear()
cluster.set_install_dir(**CCM_KWARGS)
except Exception:
log.debug("Creating new ccm %s cluster with %s", cluster_name, CCM_KWARGS)
cluster = CCMCluster(path, cluster_name, **CCM_KWARGS)
cluster.set_configuration_options({'start_native_transport': True})
if CASSANDRA_VERSION >= '2.2':
cluster.set_configuration_options({'enable_user_defined_functions': True})
common.switch_cluster(path, cluster_name)
cluster.populate(nodes, ipformat=ipformat)
if start:
log.debug("Starting ccm %s cluster", cluster_name)
cluster.start(wait_for_binary_proto=True, wait_other_notice=True)
setup_keyspace(ipformat=ipformat)
CCM_CLUSTER = cluster
except Exception:
log.exception("Failed to start ccm cluster. Removing cluster.")
remove_cluster()
call(["pkill", "-9", "-f", ".ccm"])
raise
def test1(self):
self.cluster = Cluster(CLUSTER_PATH, "test1", cassandra_version='2.0.3')
self.cluster.show(False)
self.cluster.populate(2)
self.cluster.set_partitioner("Murmur3")
self.cluster.start()
self.cluster.set_configuration_options(None, None)
self.cluster.set_configuration_options({}, True)
self.cluster.set_configuration_options({"a": "b"}, False)
[node1, node2] = self.cluster.nodelist()
node2.compact()
self.cluster.flush()
self.cluster.stop()
def setUp(self):
self.test_path = tempfile.mkdtemp(prefix=self.name)
print
print "CCM using %s" % self.test_path
# ccm setup
self.session = None
shutil.rmtree(self.test_path)
os.makedirs(self.test_path)
self.cluster = Cluster(self.test_path, "test", cassandra_version="3.0.0")
# sstable tools setup
self.uberjar_location = glob.glob("%s/../../../target/sstable-*.jar" % os.path.dirname(os.path.realpath(__file__)))[0]
print "Using sstable build located at: %s" % self.uberjar_location
def teardown_package():
for cluster_name in [CLUSTER_NAME, MULTIDC_CLUSTER_NAME]:
try:
cluster = CCMCluster.load(path, cluster_name)
try:
cluster.clear()
cluster.remove()
log.info('Cleared cluster: %s' % cluster_name)
except Exception:
log.exception('Failed to clear cluster: %s' % cluster_name)
except Exception:
log.warn('Did not find cluster: %s' % cluster_name)
def issue_150_test(self):
self.cluster = Cluster(CLUSTER_PATH, "150", cassandra_version='2.0.9')
self.cluster.populate([1, 2], use_vnodes=True)
self.cluster.start()
dcs = [node.data_center for node in self.cluster.nodelist()]
dcs.append('dc2')
node4 = Node('node4', self.cluster, True, ('127.0.0.4', 9160), ('127.0.0.4', 7000),
'7400', '2000', None)
self.cluster.add(node4, False, 'dc2')
node4.start()
dcs_2 = [node.data_center for node in self.cluster.nodelist()]
self.assertItemsEqual(dcs, dcs_2)
node4.nodetool('status')
def setUp(self):
global CURRENT_TEST
CURRENT_TEST = self.id() + self._testMethodName
# cleaning up if a previous execution didn't trigger tearDown (which
# can happen if it is interrupted by KeyboardInterrupt)
# TODO: move that part to a generic fixture
if os.path.exists(LAST_TEST_DIR):
with open(LAST_TEST_DIR) as f:
self.test_path = f.readline().strip('\n')
name = f.readline()
try:
self.cluster = Cluster.load(self.test_path, name)
# Avoid waiting too long for node to be marked down
self.__cleanup_cluster()
except IOError:
# after a restart, /tmp will be emptied so we'll get an IOError when loading the old cluster here
pass
self.cluster = self._get_cluster()
self.__setup_cobertura()
# the failure detector can be quite slow in such tests with quick start/stop
self.cluster.set_configuration_options(values={'phi_convict_threshold': 5})
timeout = 10000
if self.cluster_options is not None:
self.cluster.set_configuration_options(values=self.cluster_options)
elif self.cluster.version() < "1.2":
self.cluster.set_configuration_options(values={'rpc_timeout_in_ms': timeout})
else:
self.cluster.set_configuration_options(values={
'read_request_timeout_in_ms' : timeout,
'range_request_timeout_in_ms' : timeout,
'write_request_timeout_in_ms' : timeout,
'truncate_request_timeout_in_ms' : timeout,
'request_timeout_in_ms' : timeout
})
with open(LAST_TEST_DIR, 'w') as f:
f.write(self.test_path + '\n')
f.write(self.cluster.name)
if DEBUG:
self.cluster.set_log_level("DEBUG")
if TRACE:
self.cluster.set_log_level("TRACE")
self.connections = []
self.runners = []
def create_ccm_cluster(dtest_setup):
logger.info("cluster ccm directory: " + dtest_setup.test_path)
version = dtest_setup.dtest_config.cassandra_version
if version:
cluster = Cluster(dtest_setup.test_path, dtest_setup.cluster_name, cassandra_version=version)
else:
cluster = Cluster(dtest_setup.test_path, dtest_setup.cluster_name, cassandra_dir=dtest_setup.dtest_config.cassandra_dir)
cluster.set_datadir_count(dtest_setup.dtest_config.data_dir_count)
cluster.set_environment_variable('CASSANDRA_LIBJEMALLOC', dtest_setup.dtest_config.jemalloc_path)
return cluster
def setup_package():
try:
try:
cluster = CCMCluster.load(path, CLUSTER_NAME)
log.debug("Found existing ccm test cluster, clearing")
cluster.clear()
except Exception:
log.debug("Creating new ccm test cluster")
cluster = CCMCluster(path, CLUSTER_NAME, cassandra_version='1.2.6')
cluster.set_configuration_options({'start_native_transport': True})
common.switch_cluster(path, CLUSTER_NAME)
cluster.populate(3)
log.debug("Starting ccm test cluster")
cluster.start(wait_for_binary_proto=True)
except Exception:
log.exception("Failed to start ccm cluster:")
raise
global CCM_CLUSTER
CCM_CLUSTER = cluster
def test2(self):
self.cluster = Cluster(CLUSTER_PATH, "test2", cassandra_version='2.0.3')
self.cluster.populate(2)
self.cluster.start()
self.cluster.set_log_level("ERROR")
class FakeNode:
name = "non-existing node"
self.cluster.remove(FakeNode())
[node1, node2] = self.cluster.nodelist()
self.cluster.remove(node1)
self.cluster.show(True)
self.cluster.show(False)
#self.cluster.stress([])
self.cluster.compact()
self.cluster.drain()
self.cluster.stop()
def decomission_test(self):
cluster = self.cluster
tokens = Cluster.balanced_tokens(4)
cluster.populate(4, tokens=tokens).start()
[node1, node2, node3, node4] = cluster.nodelist()
cursor = self.cql_connection(node1).cursor()
self.create_ks(cursor, 'ks', 2)
self.create_cf(cursor, 'cf')
for n in xrange(0, 10000):
insert_c1c2(cursor, n, "QUORUM")
cluster.flush()
sizes = [ node.data_size() for node in cluster.nodelist() if node.is_running()]
init_size = sizes[0]
assert_almost_equal(*sizes)
time.sleep(.5)
node4.decommission()
node4.stop()
cluster.cleanup()
time.sleep(.5)
# Check we can get all the keys
for n in xrange(0, 10000):
query_c1c2(cursor, n, "QUORUM")
sizes = [ node.data_size() for node in cluster.nodelist() if node.is_running() ]
three_node_sizes = sizes
assert_almost_equal(sizes[0], sizes[1])
assert_almost_equal((2.0/3.0) * sizes[0], sizes[2])
assert_almost_equal(sizes[2], init_size)
node3.stop(wait_other_notice=True)
node1.removeToken(tokens[2])
time.sleep(.5)
cluster.cleanup()
time.sleep(.5)
# Check we can get all the keys
for n in xrange(0, 10000):
query_c1c2(cursor, n, "QUORUM")
sizes = [ node.data_size() for node in cluster.nodelist() if node.is_running() ]
assert_almost_equal(*sizes)
assert_almost_equal(sizes[0], 2 * init_size)
node5 = new_node(cluster, token=(tokens[2]+1)).start()
time.sleep(.5)
cluster.cleanup()
time.sleep(.5)
# Check we can get all the keys
for n in xrange(0, 10000):
query_c1c2(cursor, n, "QUORUM")
sizes = [ node.data_size() for node in cluster.nodelist() if node.is_running() ]
# We should be back to the earlir 3 nodes situation
for i in xrange(0, len(sizes)):
assert_almost_equal(sizes[i], three_node_sizes[i])
def stop(self, wait=True, gently=True, wait_other_notice=False, wait_seconds=127):
if self._scylla_manager:
self._scylla_manager.stop(gently)
Cluster.stop(self,wait,gently,wait_seconds=wait_seconds)
def run(self):
try:
if self.options.scylla:
if self.options.docker_image:
cluster = ScyllaDockerCluster(self.path, self.name, docker_image=self.options.docker_image)
else:
if self.options.scylla_manager_package:
from ccmlib.scylla_repository import setup_scylla_manager
manager_install_dir = setup_scylla_manager(self.options.scylla_manager_package)
else:
manager_install_dir = self.options.scyllamanager
cluster = ScyllaCluster(self.path, self.name, install_dir=self.options.install_dir,
version=self.options.version, verbose=True, manager=manager_install_dir)
elif self.options.dse or (not self.options.version and common.isDse(self.options.install_dir)):
cluster = DseCluster(self.path, self.name, install_dir=self.options.install_dir, version=self.options.version, dse_username=self.options.dse_username, dse_password=self.options.dse_password, opscenter=self.options.opscenter, verbose=True)
else:
cluster = Cluster(self.path, self.name, install_dir=self.options.install_dir, version=self.options.version, verbose=True)
except OSError as e:
import traceback
print_('Cannot create cluster: %s\n%s' % (str(e), traceback.format_exc()), file=sys.stderr)
sys.exit(1)
if self.options.partitioner:
cluster.set_partitioner(self.options.partitioner)
if self.options.snitch:
cluster.set_snitch(self.options.snitch)
if self.options.id:
cluster.set_id(self.options.id)
if not self.options.ipprefix:
self.options.ipprefix = "127.0.%d." % self.options.id
if cluster.cassandra_version() >= "1.2.5":
self.options.binary_protocol = True
if self.options.binary_protocol:
cluster.set_configuration_options({'start_native_transport': True})
if cluster.cassandra_version() >= "1.2" and self.options.vnodes:
cluster.set_configuration_options({'num_tokens': 256})
if not self.options.no_switch:
common.switch_cluster(self.path, self.name)
print_('Current cluster is now: %s' % self.name)
if not (self.options.ipprefix or self.options.ipformat):
self.options.ipformat = '127.0.0.%d'
if self.options.ssl_path:
cluster.enable_ssl(self.options.ssl_path, self.options.require_client_auth)
if self.options.node_ssl_path:
cluster.enable_internode_ssl(self.options.node_ssl_path)
if self.nodes is not None:
try:
if self.options.debug_log:
cluster.set_log_level("DEBUG")
if self.options.trace_log:
cluster.set_log_level("TRACE")
cluster.populate(self.nodes, self.options.debug, use_vnodes=self.options.vnodes, ipprefix=self.options.ipprefix, ipformat=self.options.ipformat)
if self.options.start_nodes:
profile_options = None
if self.options.profile:
profile_options = {}
if self.options.profile_options:
profile_options['options'] = self.options.profile_options
if cluster.start(verbose=self.options.debug_log, wait_for_binary_proto=self.options.binary_protocol, jvm_args=self.options.jvm_args, profile_options=profile_options) is None:
details = ""
if not self.options.debug_log:
details = " (you can use --debug-log for more information)"
print_("Error starting nodes, see above for details%s" % details, file=sys.stderr)
except common.ArgumentError as e:
print_(str(e), file=sys.stderr)
sys.exit(1)
def force_stop(node):
log.debug("Forcing stop of node %s", node)
get_node(node).stop(wait=False, gently=False)
log.debug("Node %s was stopped", node)
def get_node(node_id):
return CCM_CLUSTER.nodes['node%s' % node_id]
CCM_KWARGS = {}
CCM_KWARGS['version'] = "2.0.17"
path_to_ccm = "/home/jaume/.ccm/"
CCM_CLUSTER = CCMCluster(path_to_ccm, "test_for_is_up", **CCM_KWARGS)
CCM_CLUSTER.populate(3)
CCM_CLUSTER.start(wait_for_binary_proto=True,
wait_other_notice=True,
jvm_args=[])
def main():
log.info("\n\n\n\n Starting iteration\n")
node_to_stop = 3
log.info("node_to_stop: {0}".format(node_to_stop))
cluster = Cluster(protocol_version=PROTOCOL_VERSION,
connection_class=connection_class)
cluster.connect(wait_for_all_pools=True)
request_id=0,
cb=cb)
conn.close()
log.debug("\n\nRequest ended\n\n")
setup_cluster = False
if setup_cluster:
CCM_KWARGS = {}
CCM_KWARGS['version'] = "3.7"
path_to_ccm = "/home/jaume/.ccm/"
CCM_CLUSTER = CCMCluster(path_to_ccm, "test_for_is_conn", **CCM_KWARGS)
CCM_CLUSTER.populate(3)
CCM_CLUSTER.clear()
CCM_CLUSTER.start()
time.sleep(5)
connection_class.initialize_reactor()
try:
for _ in range(40000):
make_request()
except Exception as e:
import traceback
traceback.print_exc()
raise e
def test3():
cluster = Cluster(CLUSTER_PATH, "test3", cassandra_version='2.0.3')
cluster.populate(2)
cluster.start()
cluster.cleanup()
cluster.clear()
cluster.stop()
class TestRunCqlsh(ccmtest.Tester):
def setUp(self):
'''Create a cluster for cqlsh tests. Assumes that ccmtest.Tester's
teardown() method will safely stop and remove self.cluster.'''
self.cluster = Cluster(CLUSTER_PATH,
"run_cqlsh",
cassandra_version='git:trunk')
self.cluster.populate(1).start(wait_for_binary_proto=True)
[self.node] = self.cluster.nodelist()
def test_run_cqlsh(self):
'''run_cqlsh works with a simple example input'''
self.node.run_cqlsh('''
CREATE KEYSPACE ks WITH replication = { 'class' :'SimpleStrategy', 'replication_factor': 1};
USE ks;
CREATE TABLE test (key int PRIMARY KEY);
INSERT INTO test (key) VALUES (1);
''')
rv = self.node.run_cqlsh('SELECT * from ks.test', return_output=True)
for s in ['(1 rows)', 'key', '1']:
self.assertIn(s, rv[0])
self.assertEqual(rv[1], '')
def run_cqlsh_printing(self, return_output, show_output):
'''Parameterized test. Runs run_cqlsh with options to print the output
and to return it as a string, or with these options combined, depending
on the values of the arguments.'''
# redirect run_cqlsh's stdout to a string buffer
old_stdout, sys.stdout = sys.stdout, StringIO()
rv = self.node.run_cqlsh('DESCRIBE keyspaces;',
return_output=return_output,
show_output=show_output)
# put stdout back where it belongs and get the built string value
sys.stdout, printed_output = old_stdout, sys.stdout.getvalue()
if return_output:
# we should see names of system keyspaces
self.assertIn('system', rv[0])
# stderr should be empty
self.assertEqual('', rv[1])
else:
# implicitly-returned None
self.assertEqual(rv, None)
if show_output:
self.assertIn('system', printed_output)
else:
# nothing should be printed if (not show_output)
self.assertEqual(printed_output, '')
if return_output and show_output:
self.assertEqual(printed_output, rv[0])
def test_no_output(self):
self.run_cqlsh_printing(return_output=False, show_output=False)
def test_print_output(self):
self.run_cqlsh_printing(return_output=True, show_output=False)
def test_return_output(self):
self.run_cqlsh_printing(return_output=False, show_output=True)
def test_print_and_return_output(self):
self.run_cqlsh_printing(return_output=True, show_output=True)
class TestCCMLib(ccmtest.Tester):
def test2(self):
self.cluster = Cluster(CLUSTER_PATH,
"test2",
cassandra_version='git:trunk')
self.cluster.populate(2)
self.cluster.start()
self.cluster.set_log_level("ERROR")
class FakeNode:
name = "non-existing node"
self.cluster.remove(FakeNode())
[node1, node2] = self.cluster.nodelist()
self.cluster.remove(node1)
self.cluster.show(True)
self.cluster.show(False)
# self.cluster.stress([])
self.cluster.compact()
self.cluster.drain()
self.cluster.stop()
def test3(self):
self.cluster = Cluster(CLUSTER_PATH,
"test3",
cassandra_version='git:trunk')
self.cluster.populate(2)
self.cluster.start()
self.cluster.cleanup()
self.cluster.clear()
self.cluster.stop()
def test_node_start_with_non_default_timeout(self):
self.cluster = Cluster(CLUSTER_PATH,
"nodestarttimeout",
cassandra_version='git:trunk')
self.cluster.populate(1)
node = self.cluster.nodelist()[0]
try:
node.start(wait_for_binary_proto=0)
self.fail("timeout expected with 0s startup timeout")
except ccmlib.node.TimeoutError:
pass
finally:
self.cluster.cleanup()
self.cluster.clear()
self.cluster.stop()
def test_fast_error_on_cluster_start_failure(self):
self.cluster = Cluster(CLUSTER_PATH,
'clusterfaststop',
cassandra_version='git:trunk')
self.cluster.populate(1)
self.cluster.set_configuration_options({'invalid_option': 0})
self.check_log_errors = False
node = self.cluster.nodelist()[0]
start_time = time.time()
try:
self.cluster.start(use_vnodes=True)
self.assertFalse(node.is_running())
self.assertLess(time.time() - start_time, 30)
except NodeError:
self.assertLess(time.time() - start_time, 30)
finally:
self.cluster.clear()
self.cluster.stop()
def test_fast_error_on_node_start_failure(self):
self.cluster = Cluster(CLUSTER_PATH,
'nodefaststop',
cassandra_version='git:trunk')
self.cluster.populate(1)
self.cluster.set_configuration_options({'invalid_option': 0})
self.check_log_errors = False
node = self.cluster.nodelist()[0]
start_time = time.time()
try:
node.start(wait_for_binary_proto=45)
self.assertFalse(node.is_running())
self.assertLess(time.time() - start_time, 30)
except NodeError:
self.assertLess(time.time() - start_time, 30)
finally:
self.cluster.clear()
self.cluster.stop()
def test2():
cluster = Cluster(CLUSTER_PATH, "test2", cassandra_version='2.0.3')
cluster.populate(2)
cluster.start()
cluster.set_log_level("ERROR")
class FakeNode:
name = "non-existing node"
cluster.remove(FakeNode())
[node1, node2] = cluster.nodelist()
cluster.remove(node1)
cluster.show(True)
cluster.show(False)
#cluster.stress([])
cluster.compact()
cluster.drain()
cluster.stop()
class TestCCMLib(ccmtest.Tester):
def simple_test(self, version='2.0.9'):
self.cluster = Cluster(CLUSTER_PATH,
"simple",
cassandra_version=version)
self.cluster.populate(3)
self.cluster.start()
node1, node2, node3 = self.cluster.nodelist()
if version < '2.1':
node1.stress()
else:
node1.stress(['write', 'n=1000000'])
self.cluster.flush()
def simple_test_across_versions(self):
self.simple_test(version='1.2.18')
self.cluster.remove()
self.simple_test(version='2.0.9')
self.cluster.remove()
self.simple_test(version='2.1.0-rc5')
def restart_test(self):
self.cluster = Cluster(CLUSTER_PATH,
"restart",
cassandra_version='2.0.9')
self.cluster.populate(3)
self.cluster.start()
self.cluster.stop()
self.cluster.start()
self.cluster.show(True)
def multi_dc_test(self):
self.cluster = Cluster(CLUSTER_PATH,
"multi_dc",
cassandra_version='2.0.9')
self.cluster.populate([1, 2])
self.cluster.start()
dcs = [node.data_center for node in self.cluster.nodelist()]
self.cluster.set_configuration_options(None, None)
self.cluster.stop()
self.cluster.start()
dcs_2 = [node.data_center for node in self.cluster.nodelist()]
self.assertListEqual(dcs, dcs_2)
def test1(self):
self.cluster = Cluster(CLUSTER_PATH,
"test1",
cassandra_version='2.0.3')
self.cluster.show(False)
self.cluster.populate(2)
self.cluster.set_partitioner("Murmur3")
self.cluster.start()
self.cluster.set_configuration_options(None, None)
self.cluster.set_configuration_options({}, True)
self.cluster.set_configuration_options({"a": "b"}, False)
[node1, node2] = self.cluster.nodelist()
node2.compact()
self.cluster.flush()
self.cluster.stop()
def test2(self):
self.cluster = Cluster(CLUSTER_PATH,
"test2",
cassandra_version='2.0.3')
self.cluster.populate(2)
self.cluster.start()
self.cluster.set_log_level("ERROR")
class FakeNode:
name = "non-existing node"
self.cluster.remove(FakeNode())
[node1, node2] = self.cluster.nodelist()
self.cluster.remove(node1)
self.cluster.show(True)
self.cluster.show(False)
#self.cluster.stress([])
self.cluster.compact()
self.cluster.drain()
self.cluster.stop()
def test3(self):
self.cluster = Cluster(CLUSTER_PATH,
"test3",
cassandra_version='2.0.3')
self.cluster.populate(2)
self.cluster.start()
self.cluster.cleanup()
self.cluster.clear()
self.cluster.stop()
def use_cluster(cluster_name,
nodes,
ipformat=None,
start=True,
workloads=[],
set_keyspace=True,
ccm_options=None,
configuration_options={}):
set_default_cass_ip()
if ccm_options is None:
ccm_options = CCM_KWARGS
cassandra_version = ccm_options.get('version', CASSANDRA_VERSION)
global CCM_CLUSTER
if USE_CASS_EXTERNAL:
if CCM_CLUSTER:
log.debug("Using external CCM cluster {0}".format(
CCM_CLUSTER.name))
else:
log.debug("Using unnamed external cluster")
if set_keyspace and start:
setup_keyspace(ipformat=ipformat, wait=False)
return
if is_current_cluster(cluster_name, nodes):
log.debug("Using existing cluster, matching topology: {0}".format(
cluster_name))
else:
if CCM_CLUSTER:
log.debug(
"Stopping existing cluster, topology mismatch: {0}".format(
CCM_CLUSTER.name))
CCM_CLUSTER.stop()
try:
CCM_CLUSTER = CCMClusterFactory.load(path, cluster_name)
log.debug("Found existing CCM cluster, {0}; clearing.".format(
cluster_name))
CCM_CLUSTER.clear()
CCM_CLUSTER.set_install_dir(**ccm_options)
CCM_CLUSTER.set_configuration_options(configuration_options)
except Exception:
ex_type, ex, tb = sys.exc_info()
log.warning("{0}: {1} Backtrace: {2}".format(
ex_type.__name__, ex, traceback.extract_tb(tb)))
del tb
log.debug("Creating new CCM cluster, {0}, with args {1}".format(
cluster_name, ccm_options))
CCM_CLUSTER = CCMCluster(path, cluster_name, **ccm_options)
CCM_CLUSTER.set_configuration_options(
{'start_native_transport': True})
if cassandra_version >= '2.2':
CCM_CLUSTER.set_configuration_options(
{'enable_user_defined_functions': True})
if cassandra_version >= '3.0':
CCM_CLUSTER.set_configuration_options(
{'enable_scripted_user_defined_functions': True})
common.switch_cluster(path, cluster_name)
CCM_CLUSTER.set_configuration_options(configuration_options)
CCM_CLUSTER.populate(nodes, ipformat=ipformat)
try:
jvm_args = []
# This will enable the Mirroring query handler which will echo our custom payload k,v pairs back
if 'graph' not in workloads:
if PROTOCOL_VERSION >= 4:
jvm_args = [
" -Dcassandra.custom_query_handler_class=org.apache.cassandra.cql3.CustomPayloadMirroringQueryHandler"
]
if (len(workloads) > 0):
for node in CCM_CLUSTER.nodes.values():
node.set_workloads(workloads)
if start:
log.debug("Starting CCM cluster: {0}".format(cluster_name))
CCM_CLUSTER.start(wait_for_binary_proto=True,
wait_other_notice=True,
jvm_args=jvm_args)
# Added to wait for slow nodes to start up
for node in CCM_CLUSTER.nodes.values():
wait_for_node_socket(node, 120)
if set_keyspace:
setup_keyspace(ipformat=ipformat)
except Exception:
log.exception("Failed to start CCM cluster; removing cluster.")
if os.name == "nt":
if CCM_CLUSTER:
for node in six.itervalues(CCM_CLUSTER.nodes):
os.system("taskkill /F /PID " + str(node.pid))
else:
call(["pkill", "-9", "-f", ".ccm"])
remove_cluster()
raise
return CCM_CLUSTER
class TestCCMLib(ccmtest.Tester):
def test2(self):
self.cluster = Cluster(CLUSTER_PATH,
"test2",
cassandra_version='git:trunk')
self.cluster.populate(2)
self.cluster.start()
self.cluster.set_log_level("ERROR")
class FakeNode:
name = "non-existing node"
self.cluster.remove(FakeNode())
[node1, node2] = self.cluster.nodelist()
self.cluster.remove(node1)
self.cluster.show(True)
self.cluster.show(False)
# self.cluster.stress([])
self.cluster.compact()
self.cluster.drain()
self.cluster.stop()
def test3(self):
self.cluster = Cluster(CLUSTER_PATH,
"test3",
cassandra_version='git:trunk')
self.cluster.populate(2)
self.cluster.start()
self.cluster.cleanup()
self.cluster.clear()
self.cluster.stop()
def test_node_start_with_non_default_timeout(self):
self.cluster = Cluster(CLUSTER_PATH,
"nodestarttimeout",
cassandra_version='git:trunk')
self.cluster.populate(1)
node = self.cluster.nodelist()[0]
try:
node.start(wait_for_binary_proto=0)
self.fail("timeout expected with 0s startup timeout")
except ccmlib.node.TimeoutError:
pass
finally:
self.cluster.cleanup()
self.cluster.clear()
self.cluster.stop()
def create_ccm_cluster(dtest_setup):
logger.debug("cluster ccm directory: " + dtest_setup.test_path)
version = dtest_setup.dtest_config.cassandra_version
if version:
cluster = Cluster(dtest_setup.test_path, dtest_setup.cluster_name, cassandra_version=version)
else:
cluster = Cluster(dtest_setup.test_path, dtest_setup.cluster_name, cassandra_dir=dtest_setup.dtest_config.cassandra_dir)
if dtest_setup.dtest_config.use_vnodes:
cluster.set_configuration_options(values={'initial_token': None, 'num_tokens': dtest_setup.dtest_config.num_tokens})
else:
cluster.set_configuration_options(values={'num_tokens': None})
if dtest_setup.dtest_config.use_off_heap_memtables:
cluster.set_configuration_options(values={'memtable_allocation_type': 'offheap_objects'})
cluster.set_datadir_count(dtest_setup.dtest_config.data_dir_count)
cluster.set_environment_variable('CASSANDRA_LIBJEMALLOC', dtest_setup.dtest_config.jemalloc_path)
return cluster
def run(self):
try:
cluster = Cluster(self.path,
self.name,
cassandra_dir=self.options.cassandra_dir,
cassandra_version=self.options.cassandra_version,
verbose=True)
except OSError as e:
cluster_dir = os.path.join(self.path, self.name)
import traceback
print_('Cannot create cluster: %s\n%s' %
(str(e), traceback.format_exc()),
file=sys.stderr)
exit(1)
if self.options.partitioner:
cluster.set_partitioner(self.options.partitioner)
if cluster.version() >= "1.2.5":
self.options.binary_protocol = True
if self.options.binary_protocol:
cluster.set_configuration_options({'start_native_transport': True})
if cluster.version() >= "1.2" and self.options.vnodes:
cluster.set_configuration_options({'num_tokens': 256})
if not self.options.no_switch:
common.switch_cluster(self.path, self.name)
print_('Current cluster is now: %s' % self.name)
if not (self.options.ipprefix or self.options.ipformat):
self.options.ipformat = '127.0.0.%d'
if self.nodes is not None:
try:
if self.options.debug_log:
cluster.set_log_level("DEBUG")
if self.options.trace_log:
cluster.set_log_level("TRACE")
cluster.populate(self.nodes,
use_vnodes=self.options.vnodes,
ipprefix=self.options.ipprefix,
ipformat=self.options.ipformat)
if self.options.start_nodes:
profile_options = None
if self.options.profile:
profile_options = {}
if self.options.profile_options:
profile_options[
'options'] = self.options.profile_options
if cluster.start(
verbose=self.options.debug,
wait_for_binary_proto=self.options.binary_protocol,
jvm_args=self.options.jvm_args,
profile_options=profile_options) is None:
details = ""
if not self.options.debug:
details = " (you can use --debug for more information)"
print_(
"Error starting nodes, see above for details%s" %
details,
file=sys.stderr)
except common.ArgumentError as e:
print_(str(e), file=sys.stderr)
exit(1)
class TestCCMLib(ccmtest.Tester):
def test2(self):
self.cluster = Cluster(CLUSTER_PATH,
"test2",
cassandra_version='git:trunk')
self.cluster.populate(2)
self.cluster.start()
self.cluster.set_log_level("ERROR")
class FakeNode:
name = "non-existing node"
self.cluster.remove(FakeNode())
[node1, node2] = self.cluster.nodelist()
self.cluster.remove(node1)
self.cluster.show(True)
self.cluster.show(False)
# self.cluster.stress([])
self.cluster.compact()
self.cluster.drain()
self.cluster.stop()
def test3(self):
self.cluster = Cluster(CLUSTER_PATH,
"test3",
cassandra_version='git:trunk')
self.cluster.populate(2)
self.cluster.start()
self.cluster.cleanup()
self.cluster.clear()
self.cluster.stop()
def setup_class(cls):
"""
Class-level fixture. Called once for all tests.
We create a cluster object here and save it in the provided class instance
Create 2 regular users and a new superuser.
Enable authentication by setting 'authenticator': 'PasswordAuthenticator'.
If invoked from authorization class enable authorization by setting 'authorizer': 'CassandraAuthorizer'.
"""
try:
ccm_cluster = CCMCluster.load(tests.integration.path,
AUTH_CLUSTER_NAME)
log.debug(
"Found existing ccm test authentication cluster, removing")
ccm_cluster.remove()
except Exception:
log.debug("Can not load cluster %s ....." % AUTH_CLUSTER_NAME)
log.debug("Creating new ccm test authentication cluster")
if tests.integration.CASSANDRA_DIR:
ccm_cluster = CCMCluster(
tests.integration.path,
AUTH_CLUSTER_NAME,
cassandra_dir=tests.integration.CASSANDRA_DIR)
else:
ccm_cluster = CCMCluster(
tests.integration.path,
AUTH_CLUSTER_NAME,
cassandra_version=tests.integration.CASSANDRA_VERSION)
ccm_cluster.set_configuration_options({'start_native_transport': True})
ccm_cluster.set_configuration_options(
{'authenticator': 'PasswordAuthenticator'})
#
# This method is called either with AuthenticationTests class or with AuthorizedAuthenticationTests class.
# In the second case we enable CassandraAuthorizer
#
if cls.__name__ == 'AuthorizedAuthenticationTests':
print "Running tests with Cassandra Authorizer Enabled"
log.info("Running tests with Cassandra Authorizer Enabled")
ccm_cluster.set_configuration_options(
{'authorizer': 'CassandraAuthorizer'})
else:
print "Running tests with Cassandra Authorizer Disabled"
log.info("Running tests with Cassandra Authorizer Disabled")
common.switch_cluster(tests.integration.path, AUTH_CLUSTER_NAME)
ccm_cluster.populate(1)
log.debug("Starting ccm test authentication cluster")
ccm_cluster.start(wait_for_binary_proto=True)
if not wait_for_cassandra() or not try_connecting(
'cassandra', 'cassandra'):
log.exception("Can not talk to cassandra")
raise Exception('Can not talk to cassandra')
log.debug("Switched to AUTH_CLUSTER_NAME cluster")
cls.ccm_cluster = ccm_cluster
cls.root_cluster = cls.cluster_as('cassandra', 'cassandra')
cls.root_session = cls.root_cluster.connect()
cls.create_user(cls.root_cluster, cls.test_user, cls.password)
cls.create_user(cls.root_cluster, cls.test_other_user, cls.password)
cls.create_user(cls.root_cluster, cls.suser, cls.password, su=True)
def test1():
cluster = Cluster(CLUSTER_PATH, "test1", cassandra_version='2.0.3')
cluster.show(False)
cluster.populate(2)
cluster.set_partitioner("Murmur3")
cluster.start()
cluster.set_configuration_options(None, None)
cluster.set_configuration_options({}, True)
cluster.set_configuration_options({"a": "b"}, False)
[node1, node2] = cluster.nodelist()
node2.compact()
cluster.flush()
cluster.remove()
cluster.stop()
def run(self):
try:
cluster = Cluster(self.path,
self.name,
cassandra_dir=self.options.cassandra_dir,
cassandra_version=self.options.cassandra_version,
verbose=True)
except OSError as e:
cluster_dir = os.path.join(self.path, self.name)
import traceback
print >> sys.stderr, 'Cannot create cluster: %s\n%s' % (
str(e), traceback.format_exc())
exit(1)
if self.options.partitioner:
cluster.set_partitioner(self.options.partitioner)
if cluster.version() >= "1.2" and self.options.binary_protocol:
cluster.set_configuration_options({'start_native_transport': True})
if cluster.version() >= "1.2" and self.options.vnodes:
cluster.set_configuration_options({'num_tokens': 256})
if not self.options.no_switch:
common.switch_cluster(self.path, self.name)
print 'Current cluster is now: %s' % self.name
if self.nodes is not None:
try:
if self.options.debug_log:
cluster.set_log_level("DEBUG")
if self.options.trace_log:
cluster.set_log_level("TRACE")
cluster.populate(self.nodes,
use_vnodes=self.options.vnodes,
ipprefix=self.options.ipprefix)
if self.options.start_nodes:
cluster.start(
verbose=self.options.debug,
wait_for_binary_proto=self.options.binary_protocol)
except common.ArgumentError as e:
print >> sys.stderr, str(e)
exit(1)
def run(self):
try:
if self.options.dse or (not self.options.version and common.isDse(self.options.install_dir)):
cluster = DseCluster(self.path, self.name, install_dir=self.options.install_dir, version=self.options.version, dse_username=self.options.dse_username, dse_password=self.options.dse_password, dse_credentials_file=self.options.dse_credentials_file, opscenter=self.options.opscenter, verbose=True)
else:
cluster = Cluster(self.path, self.name, install_dir=self.options.install_dir, version=self.options.version, verbose=True)
except OSError as e:
import traceback
print_('Cannot create cluster: %s\n%s' % (str(e), traceback.format_exc()), file=sys.stderr)
exit(1)
if self.options.partitioner:
cluster.set_partitioner(self.options.partitioner)
if cluster.cassandra_version() >= "1.2.5":
self.options.binary_protocol = True
if self.options.binary_protocol:
cluster.set_configuration_options({'start_native_transport': True})
if cluster.cassandra_version() >= "1.2" and self.options.vnodes:
cluster.set_configuration_options({'num_tokens': 256})
if not self.options.no_switch:
common.switch_cluster(self.path, self.name)
print_('Current cluster is now: %s' % self.name)
if not (self.options.ipprefix or self.options.ipformat):
self.options.ipformat = '127.0.0.%d'
if self.options.ssl_path:
cluster.enable_ssl(self.options.ssl_path, self.options.require_client_auth)
if self.options.node_ssl_path:
cluster.enable_internode_ssl(self.options.node_ssl_path)
if self.options.node_pwd_auth:
cluster.enable_pwd_auth()
if self.options.datadirs:
cluster.set_datadir_count(self.options.datadirs)
if self.nodes is not None:
try:
if self.options.debug_log:
cluster.set_log_level("DEBUG")
if self.options.trace_log:
cluster.set_log_level("TRACE")
cluster.populate(self.nodes, self.options.debug, use_vnodes=self.options.vnodes, ipprefix=self.options.ipprefix, ipformat=self.options.ipformat, install_byteman=self.options.install_byteman)
if self.options.start_nodes:
profile_options = None
if self.options.profile:
profile_options = {}
if self.options.profile_options:
profile_options['options'] = self.options.profile_options
if cluster.start(verbose=self.options.debug, wait_for_binary_proto=self.options.binary_protocol, jvm_args=self.options.jvm_args, profile_options=profile_options, allow_root=self.options.allow_root) is None:
details = ""
if not self.options.debug_log:
details = " (you can use --debug for more information)"
print_("Error starting nodes, see above for details%s" % details, file=sys.stderr)
except common.ArgumentError as e:
print_(str(e), file=sys.stderr)
exit(1)
def _get_cluster(self, name='test'):
if self._preserve_cluster and hasattr(self, 'cluster'):
return self.cluster
self.test_path = tempfile.mkdtemp(prefix='dtest-')
# ccm on cygwin needs absolute path to directory - it crosses from cygwin space into
# regular Windows space on wmic calls which will otherwise break pathing
if sys.platform == "cygwin":
self.test_path = subprocess.Popen(
["cygpath", "-m", self.test_path],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT).communicate()[0].rstrip()
debug("cluster ccm directory: " + self.test_path)
version = os.environ.get('CASSANDRA_VERSION')
cdir = CASSANDRA_DIR
if version:
cluster = Cluster(self.test_path, name, cassandra_version=version)
else:
cluster = Cluster(self.test_path, name, cassandra_dir=cdir)
if DISABLE_VNODES:
cluster.set_configuration_options(values={'num_tokens': None})
else:
cluster.set_configuration_options(values={
'initial_token': None,
'num_tokens': NUM_TOKENS
})
if OFFHEAP_MEMTABLES:
cluster.set_configuration_options(
values={'memtable_allocation_type': 'offheap_objects'})
cluster.set_datadir_count(DATADIR_COUNT)
return cluster
def create_ccm_cluster(test_path, name):
debug("cluster ccm directory: " + test_path)
version = os.environ.get('CASSANDRA_VERSION')
cdir = CASSANDRA_DIR
if version:
cluster = Cluster(test_path, name, cassandra_version=version)
else:
cluster = Cluster(test_path, name, cassandra_dir=cdir)
if DISABLE_VNODES:
cluster.set_configuration_options(values={'num_tokens': None})
else:
cluster.set_configuration_options(values={
'initial_token': None,
'num_tokens': NUM_TOKENS
})
if OFFHEAP_MEMTABLES:
cluster.set_configuration_options(
values={'memtable_allocation_type': 'offheap_objects'})
cluster.set_datadir_count(DATADIR_COUNT)
cluster.set_environment_variable('CASSANDRA_LIBJEMALLOC',
CASSANDRA_LIBJEMALLOC)
return cluster
class TestCCMLib(ccmtest.Tester):
def test2(self):
self.cluster = Cluster(CLUSTER_PATH, "test2", version='git:trunk')
self.cluster.populate(2)
self.cluster.start()
self.cluster.set_log_level("ERROR")
class FakeNode:
name = "non-existing node"
self.cluster.remove(FakeNode())
node1 = self.cluster.nodelist()[0]
self.cluster.remove(node1)
self.cluster.show(True)
self.cluster.show(False)
self.cluster.compact()
self.cluster.drain()
self.cluster.stop()
def test3(self):
self.cluster = Cluster(CLUSTER_PATH, "test3", version='git:trunk')
self.cluster.populate(2)
self.cluster.start()
node1 = self.cluster.nodelist()[0]
self.cluster.stress(
['write', 'n=100', 'no-warmup', '-rate', 'threads=4'])
self.cluster.stress([
'write', 'n=100', 'no-warmup', '-rate', 'threads=4', '-node',
node1.ip_addr
])
self.cluster.cleanup()
self.cluster.clear()
self.cluster.stop()
def test_node_start_with_non_default_timeout(self):
self.cluster = Cluster(CLUSTER_PATH,
"nodestarttimeout",
cassandra_version='git:trunk')
self.cluster.populate(1)
node = self.cluster.nodelist()[0]
try:
node.start(wait_for_binary_proto=0)
self.fail("timeout expected with 0s startup timeout")
except ccmlib.node.TimeoutError:
pass
finally:
self.cluster.cleanup()
self.cluster.clear()
self.cluster.stop()
def test_fast_error_on_cluster_start_failure(self):
self.cluster = Cluster(CLUSTER_PATH,
'clusterfaststop',
cassandra_version='git:trunk')
self.cluster.populate(1)
self.cluster.set_configuration_options({'invalid_option': 0})
self.check_log_errors = False
node = self.cluster.nodelist()[0]
start_time = time.time()
try:
self.cluster.start(use_vnodes=True)
self.assertFalse(node.is_running())
self.assertLess(time.time() - start_time, 30)
except NodeError:
self.assertLess(time.time() - start_time, 30)
finally:
self.cluster.clear()
self.cluster.stop()
def test_fast_error_on_node_start_failure(self):
self.cluster = Cluster(CLUSTER_PATH,
'nodefaststop',
cassandra_version='git:trunk')
self.cluster.populate(1)
self.cluster.set_configuration_options({'invalid_option': 0})
self.check_log_errors = False
node = self.cluster.nodelist()[0]
start_time = time.time()
try:
node.start(wait_for_binary_proto=45)
self.assertFalse(node.is_running())
self.assertLess(time.time() - start_time, 30)
except NodeError:
self.assertLess(time.time() - start_time, 30)
finally:
self.cluster.clear()
self.cluster.stop()
def test_dc_mandatory_on_multidc(self):
self.cluster = Cluster(CLUSTER_PATH,
"mandatorydc",
cassandra_version='git:trunk')
self.cluster.populate([1, 1])
node3 = self.cluster.create_node(name='node3',
auto_bootstrap=True,
thrift_interface=('127.0.0.3', 9160),
storage_interface=('127.0.0.3', 7000),
jmx_port='7300',
remote_debug_port='0',
initial_token=None,
binary_interface=('127.0.0.3', 9042))
with self.assertRaisesRegexp(
ccmlib.common.ArgumentError,
'Please specify the DC this node should be added to'):
self.cluster.add(node3, is_seed=False)
