def _populateCluster(self,
enableSSL=False,
nativePort=None,
nativePortSSL=None):
cluster = self.cluster
if enableSSL:
generate_ssl_stores(self.test_path)
cluster.set_configuration_options({
'client_encryption_options': {
'enabled': True,
'keystore': os.path.join(self.test_path, 'keystore.jks'),
'keystore_password': '******'
}
})
if nativePort:
cluster.set_configuration_options(
{'native_transport_port': nativePort})
if nativePortSSL:
cluster.set_configuration_options(
{'native_transport_port_ssl': nativePortSSL})
cluster.populate(1)
return cluster
def _populateCluster(self, enableSSL=False, nativePort=None, nativePortSSL=None):
cluster = self.cluster
if enableSSL:
generate_ssl_stores(self.test_path)
cluster.set_configuration_options({
'client_encryption_options': {
'enabled': True,
'keystore': os.path.join(self.test_path, 'keystore.jks'),
'keystore_password': '******'
}
})
if nativePort:
cluster.set_configuration_options({
'native_transport_port': nativePort
})
if nativePortSSL:
cluster.set_configuration_options({
'native_transport_port_ssl': nativePortSSL
})
cluster.populate(1)
return cluster
def __putget_with_internode_ssl_test(self, internode_compression):
cluster = self.cluster
debug("***using internode ssl***")
generate_ssl_stores(self.test_path)
cluster.set_configuration_options({'internode_compression': internode_compression})
cluster.enable_internode_ssl(self.test_path)
cluster.populate(3).start()
session = self.patient_cql_connection(cluster.nodelist()[0])
self.create_ks(session, 'ks', 3)
self.create_cf(session, 'cf', compression=None)
putget(cluster, session)
def __putget_with_internode_ssl_test(self, internode_compression):
cluster = self.cluster
debug("***using internode ssl***")
generate_ssl_stores(self.test_path)
cluster.set_configuration_options({'internode_compression': internode_compression})
cluster.enable_internode_ssl(self.test_path)
cluster.populate(3).start()
session = self.patient_cql_connection(cluster.nodelist()[0])
self.create_ks(session, 'ks', 3)
self.create_cf(session, 'cf', compression=None)
putget(cluster, session)
def _populateCluster(self, require_client_auth=False):
cluster = self.cluster
cluster.populate(1)
generate_ssl_stores(self.test_path)
if require_client_auth:
ts = self.truststore()
ts_pwd = self.truststore_password
else:
ts = None
ts_pwd = None
enable_jmx_ssl(cluster.nodelist()[0],
require_client_auth=require_client_auth,
keystore=self.keystore(),
keystore_password=self.keystore_password,
truststore=ts,
truststore_password=ts_pwd)
return cluster
def _populateCluster(self, require_client_auth=False):
cluster = self.cluster
cluster.populate(1)
generate_ssl_stores(self.test_path)
if require_client_auth:
ts = self.truststore()
ts_pwd = self.truststore_password
else:
ts = None
ts_pwd = None
enable_jmx_ssl(cluster.nodelist()[0],
require_client_auth=require_client_auth,
keystore=self.keystore(),
keystore_password=self.keystore_password,
truststore=ts,
truststore_password=ts_pwd)
return cluster
def _populateCluster(self,
enableSSL=False,
nativePort=None,
nativePortSSL=None,
sslOptional=False):
cluster = self.cluster
if enableSSL:
generate_ssl_stores(self.test_path)
# C* versions before 3.0 (CASSANDRA-10559) do not know about
# 'client_encryption_options.optional' - so we must not add that parameter
if sslOptional:
cluster.set_configuration_options({
'client_encryption_options': {
'enabled': True,
'optional': sslOptional,
'keystore': os.path.join(self.test_path,
'keystore.jks'),
'keystore_password': '******'
}
})
else:
cluster.set_configuration_options({
'client_encryption_options': {
'enabled': True,
'keystore': os.path.join(self.test_path,
'keystore.jks'),
'keystore_password': '******'
}
})
if nativePort:
cluster.set_configuration_options(
{'native_transport_port': nativePort})
if nativePortSSL:
cluster.set_configuration_options(
{'native_transport_port_ssl': nativePortSSL})
cluster.populate(1)
return cluster
def _populateCluster(self, enableSSL=False, nativePort=None, nativePortSSL=None):
cluster = self.cluster
if enableSSL:
generate_ssl_stores(self.test_path)
cluster.set_configuration_options(
{
"client_encryption_options": {
"enabled": True,
"keystore": os.path.join(self.test_path, "keystore.jks"),
"keystore_password": "******",
}
}
)
if nativePort:
cluster.set_configuration_options({"native_transport_port": nativePort})
if nativePortSSL:
cluster.set_configuration_options({"native_transport_port_ssl": nativePortSSL})
cluster.populate(1)
return cluster
def _populateCluster(self, enableSSL=False, nativePort=None, nativePortSSL=None, sslOptional=False):
cluster = self.cluster
if enableSSL:
generate_ssl_stores(self.test_path)
# C* versions before 3.0 (CASSANDRA-10559) do not know about
# 'client_encryption_options.optional' - so we must not add that parameter
if sslOptional:
cluster.set_configuration_options({
'client_encryption_options': {
'enabled': True,
'optional': sslOptional,
'keystore': os.path.join(self.test_path, 'keystore.jks'),
'keystore_password': '******'
}
})
else:
cluster.set_configuration_options({
'client_encryption_options': {
'enabled': True,
'keystore': os.path.join(self.test_path, 'keystore.jks'),
'keystore_password': '******'
}
})
if nativePort:
cluster.set_configuration_options({
'native_transport_port': nativePort
})
if nativePortSSL:
cluster.set_configuration_options({
'native_transport_port_ssl': nativePortSSL
})
cluster.populate(1)
return cluster
def upgrade_scenario(self, populate=True, create_schema=True, rolling=False, after_upgrade_call=(), internode_ssl=False):
# Record the rows we write as we go:
self.row_values = set()
cluster = self.cluster
if cluster.version() >= '3.0':
cluster.set_configuration_options({'enable_user_defined_functions': 'true',
'enable_scripted_user_defined_functions': 'true'})
elif cluster.version() >= '2.2':
cluster.set_configuration_options({'enable_user_defined_functions': 'true'})
if internode_ssl:
debug("***using internode ssl***")
generate_ssl_stores(self.test_path)
self.cluster.enable_internode_ssl(self.test_path)
if populate:
# Start with 3 node cluster
debug('Creating cluster (%s)' % self.test_version_metas[0].version)
cluster.populate(3)
[node.start(use_jna=True, wait_for_binary_proto=True) for node in cluster.nodelist()]
else:
debug("Skipping cluster creation (should already be built)")
# add nodes to self for convenience
for i, node in enumerate(cluster.nodelist(), 1):
node_name = 'node' + str(i)
setattr(self, node_name, node)
if create_schema:
if rolling:
self._create_schema_for_rolling()
else:
self._create_schema()
else:
debug("Skipping schema creation (should already be built)")
time.sleep(5) # sigh...
self._log_current_ver(self.test_version_metas[0])
if rolling:
# start up processes to write and verify data
write_proc, verify_proc, verification_queue = self._start_continuous_write_and_verify(wait_for_rowcount=5000)
increment_proc, incr_verify_proc, incr_verify_queue = self._start_continuous_counter_increment_and_verify(wait_for_rowcount=5000)
# upgrade through versions
for version_meta in self.test_version_metas[1:]:
for num, node in enumerate(self.cluster.nodelist()):
# sleep (sigh) because driver needs extra time to keep up with topo and make quorum possible
# this is ok, because a real world upgrade would proceed much slower than this programmatic one
# additionally this should provide more time for timeouts and other issues to crop up as well, which we could
# possibly "speed past" in an overly fast upgrade test
time.sleep(60)
self.upgrade_to_version(version_meta, partial=True, nodes=(node,))
self._check_on_subprocs(self.subprocs)
debug('Successfully upgraded %d of %d nodes to %s' %
(num + 1, len(self.cluster.nodelist()), version_meta.version))
self.cluster.set_install_dir(version=version_meta.version)
# Stop write processes
write_proc.terminate()
increment_proc.terminate()
# wait for the verification queue's to empty (and check all rows) before continuing
self._wait_until_queue_condition('writes pending verification', verification_queue, operator.le, 0, max_wait_s=1200)
self._wait_until_queue_condition('counters pending verification', incr_verify_queue, operator.le, 0, max_wait_s=1200)
self._check_on_subprocs([verify_proc, incr_verify_proc]) # make sure the verification processes are running still
self._terminate_subprocs()
# not a rolling upgrade, do everything in parallel:
else:
# upgrade through versions
for version_meta in self.test_version_metas[1:]:
self._write_values()
self._increment_counters()
self.upgrade_to_version(version_meta)
self.cluster.set_install_dir(version=version_meta.version)
self._check_values()
self._check_counters()
self._check_select_count()
# run custom post-upgrade callables
for call in after_upgrade_call:
call()
debug('All nodes successfully upgraded to %s' % version_meta.version)
self._log_current_ver(version_meta)
cluster.stop()
def upgrade_scenario(self, populate=True, create_schema=True, mixed_version=False, after_upgrade_call=(), internode_ssl=False):
# Record the rows we write as we go:
self.row_values = set()
cluster = self.cluster
if internode_ssl:
debug("***using internode ssl***")
generate_ssl_stores(self.test_path)
self.cluster.enable_internode_ssl(self.test_path)
if populate:
# Start with 3 node cluster
debug('Creating cluster (%s)' % self.test_versions[0])
cluster.populate(3)
[node.start(use_jna=True, wait_for_binary_proto=True) for node in cluster.nodelist()]
else:
debug("Skipping cluster creation (should already be built)")
# add nodes to self for convenience
for i, node in enumerate(cluster.nodelist(), 1):
node_name = 'node' + str(i)
setattr(self, node_name, node)
if create_schema:
self._create_schema()
else:
debug("Skipping schema creation (should already be built)")
time.sleep(5) # sigh...
self._log_current_ver(self.test_versions[0])
# upgrade through versions
for tag in self.test_versions[1:]:
if mixed_version:
for num, node in enumerate(self.cluster.nodelist()):
# do a write and check for each new node as upgraded
self._write_values()
self._increment_counters()
self.upgrade_to_version(tag, mixed_version=True, nodes=(node,))
self._check_values()
self._check_counters()
self._check_select_count()
debug('Successfully upgraded %d of %d nodes to %s' %
(num + 1, len(self.cluster.nodelist()), tag))
self.cluster.set_install_dir(version='git:' + tag)
else:
self._write_values()
self._increment_counters()
self.upgrade_to_version(tag)
self.cluster.set_install_dir(version='git:' + tag)
self._check_values()
self._check_counters()
self._check_select_count()
# run custom post-upgrade callables
for call in after_upgrade_call:
call()
debug('All nodes successfully upgraded to %s' % tag)
self._log_current_ver(tag)
cluster.stop()
def upgrade_scenario(self,
populate=True,
create_schema=True,
rolling=False,
after_upgrade_call=(),
internode_ssl=False):
# Record the rows we write as we go:
self.row_values = set()
cluster = self.cluster
if cluster.version() >= '3.0':
cluster.set_configuration_options({
'enable_user_defined_functions':
'true',
'enable_scripted_user_defined_functions':
'true'
})
elif cluster.version() >= '2.2':
cluster.set_configuration_options(
{'enable_user_defined_functions': 'true'})
if internode_ssl:
debug("***using internode ssl***")
generate_ssl_stores(self.test_path)
self.cluster.enable_internode_ssl(self.test_path)
if populate:
# Start with 3 node cluster
debug('Creating cluster (%s)' % self.test_versions[0])
cluster.populate(3)
[
node.start(use_jna=True, wait_for_binary_proto=True)
for node in cluster.nodelist()
]
else:
debug("Skipping cluster creation (should already be built)")
# add nodes to self for convenience
for i, node in enumerate(cluster.nodelist(), 1):
node_name = 'node' + str(i)
setattr(self, node_name, node)
if create_schema:
if rolling:
self._create_schema_for_rolling()
else:
self._create_schema()
else:
debug("Skipping schema creation (should already be built)")
time.sleep(5) # sigh...
self._log_current_ver(self.test_versions[0])
self.created_metadata_versions = []
if rolling:
# start up processes to write and verify data
write_proc, verify_proc, verification_queue = self._start_continuous_write_and_verify(
wait_for_rowcount=5000)
increment_proc, incr_verify_proc, incr_verify_queue = self._start_continuous_counter_increment_and_verify(
wait_for_rowcount=5000)
# upgrade through versions
for tag in self.test_versions[1:]:
for num, node in enumerate(self.cluster.nodelist()):
# sleep (sigh) because driver needs extra time to keep up with topo and make quorum possible
# this is ok, because a real world upgrade would proceed much slower than this programmatic one
# additionally this should provide more time for timeouts and other issues to crop up as well, which we could
# possibly "speed past" in an overly fast upgrade test
time.sleep(60)
self.upgrade_to_version(tag, partial=True, nodes=(node, ))
self._check_on_subprocs(self.subprocs)
debug('Successfully upgraded %d of %d nodes to %s' %
(num + 1, len(self.cluster.nodelist()), tag))
self.cluster.set_install_dir(version='git:' + tag)
# Stop write processes
write_proc.terminate()
increment_proc.terminate()
# wait for the verification queue's to empty (and check all rows) before continuing
self._wait_until_queue_condition('writes pending verification',
verification_queue,
operator.le,
0,
max_wait_s=600)
self._wait_until_queue_condition('counters pending verification',
incr_verify_queue,
operator.le,
0,
max_wait_s=600)
self._check_on_subprocs([
verify_proc, incr_verify_proc
]) # make sure the verification processes are running still
self._terminate_subprocs()
# not a rolling upgrade, do everything in parallel:
else:
# upgrade through versions
for tag in self.test_versions[1:]:
self._write_values()
self._increment_counters()
for completed_version in self.created_metadata_versions:
self._check_metadata_schemas(completed_version[0],
completed_version[1])
self._create_metadata_schemas(
self.test_versions[self.test_versions.index(tag) - 1])
self.upgrade_to_version(tag)
self.cluster.set_install_dir(version='git:' + tag)
self._check_values()
self._check_counters()
self._check_select_count()
for completed_version in self.created_metadata_versions:
self._check_metadata_schemas(completed_version[0],
completed_version[1])
# run custom post-upgrade callables
for call in after_upgrade_call:
call()
debug('All nodes successfully upgraded to %s' % tag)
self._log_current_ver(tag)
cluster.stop()
