def test_profile_pool_management(self):
"""
Tests that changes to execution profiles correctly impact our cluster's pooling
@since 3.5
@jira_ticket PYTHON-569
@expected_result pools should be correctly updated as EP's are added and removed
@test_category config_profiles
"""
node1 = ExecutionProfile(
load_balancing_policy=WhiteListRoundRobinPolicy(['127.0.0.1']))
node2 = ExecutionProfile(
load_balancing_policy=WhiteListRoundRobinPolicy(['127.0.0.2']))
with Cluster(execution_profiles={
EXEC_PROFILE_DEFAULT: node1,
'node2': node2
}) as cluster:
session = cluster.connect(wait_for_all_pools=True)
pools = session.get_pool_state()
# there are more hosts, but we connected to the ones in the lbp aggregate
self.assertGreater(len(cluster.metadata.all_hosts()), 2)
self.assertEqual(set(h.address for h in pools),
set(('127.0.0.1', '127.0.0.2')))
# dynamically update pools on add
node3 = ExecutionProfile(
load_balancing_policy=WhiteListRoundRobinPolicy(['127.0.0.3']))
cluster.add_execution_profile('node3', node3)
pools = session.get_pool_state()
self.assertEqual(set(h.address for h in pools),
set(('127.0.0.1', '127.0.0.2', '127.0.0.3')))
def test_graph_profile(self):
"""
Test verifying various aspects of graph config properties.
@since 1.0.0
@jira_ticket PYTHON-570
@test_category dse graph
"""
hosts = self.cluster.metadata.all_hosts()
first_host = hosts[0].address
second_hosts = "1.2.3.4"
generate_classic(self.session)
# Create variou execution policies
exec_dif_factory = GraphExecutionProfile(row_factory=single_object_row_factory)
exec_dif_factory.graph_options.graph_name = self.graph_name
exec_dif_lbp = GraphExecutionProfile(load_balancing_policy=WhiteListRoundRobinPolicy([first_host]))
exec_dif_lbp.graph_options.graph_name = self.graph_name
exec_bad_lbp = GraphExecutionProfile(load_balancing_policy=WhiteListRoundRobinPolicy([second_hosts]))
exec_dif_lbp.graph_options.graph_name = self.graph_name
exec_short_timeout = GraphExecutionProfile(request_timeout=1, load_balancing_policy=WhiteListRoundRobinPolicy([first_host]))
exec_short_timeout.graph_options.graph_name = self.graph_name
# Add a single exection policy on cluster creation
local_cluster = Cluster(protocol_version=PROTOCOL_VERSION, execution_profiles={"exec_dif_factory": exec_dif_factory})
local_session = local_cluster.connect()
rs1 = self.session.execute_graph('g.V()')
rs2 = local_session.execute_graph('g.V()', execution_profile='exec_dif_factory')
# Verify default and non default policy works
self.assertFalse(isinstance(rs2[0], Vertex))
self.assertTrue(isinstance(rs1[0], Vertex))
# Add other policies validate that lbp are honored
local_cluster.add_execution_profile("exec_dif_ldp", exec_dif_lbp)
local_session.execute_graph('g.V()', execution_profile="exec_dif_ldp")
local_cluster.add_execution_profile("exec_bad_lbp", exec_bad_lbp)
with self.assertRaises(NoHostAvailable):
local_session.execute_graph('g.V()', execution_profile="exec_bad_lbp")
# Try with missing EP
with self.assertRaises(ValueError):
local_session.execute_graph('g.V()', execution_profile='bad_exec_profile')
# Validate that timeout is honored
local_cluster.add_execution_profile("exec_short_timeout", exec_short_timeout)
with self.assertRaises(Exception) as e:
self.assertTrue(isinstance(e, InvalidRequest) or isinstance(e, OperationTimedOut))
local_session.execute_graph('java.util.concurrent.TimeUnit.MILLISECONDS.sleep(2000L);', execution_profile='exec_short_timeout')
def setUp(self):
self.cluster = Cluster(
protocol_version=PROTOCOL_VERSION,
execution_profiles={
EXEC_PROFILE_DEFAULT:
ExecutionProfile(
load_balancing_policy=WhiteListRoundRobinPolicy([DSE_IP]))
})
self.session = self.cluster.connect()
def test_profile_load_balancing(self):
"""
Tests that profile load balancing policies are honored.
@since 3.5
@jira_ticket PYTHON-569
@expected_result Execution Policy should be used when applicable.
@test_category config_profiles
"""
query = "select release_version from system.local"
node1 = ExecutionProfile(
load_balancing_policy=WhiteListRoundRobinPolicy([DSE_IP]))
with Cluster(execution_profiles={'node1': node1}) as cluster:
session = cluster.connect(wait_for_all_pools=True)
# default is DCA RR for all hosts
expected_hosts = set(cluster.metadata.all_hosts())
queried_hosts = set()
for _ in expected_hosts:
rs = session.execute(query)
queried_hosts.add(rs.response_future._current_host)
self.assertEqual(queried_hosts, expected_hosts)
# by name we should only hit the one
expected_hosts = set(h for h in cluster.metadata.all_hosts()
if h.address == DSE_IP)
queried_hosts = set()
for _ in cluster.metadata.all_hosts():
rs = session.execute(query, execution_profile='node1')
queried_hosts.add(rs.response_future._current_host)
self.assertEqual(queried_hosts, expected_hosts)
# use a copied instance and override the row factory
# assert last returned value can be accessed as a namedtuple so we can prove something different
named_tuple_row = rs[0]
self.assertIsInstance(named_tuple_row, tuple)
self.assertTrue(named_tuple_row.release_version)
tmp_profile = copy(node1)
tmp_profile.row_factory = tuple_factory
queried_hosts = set()
for _ in cluster.metadata.all_hosts():
rs = session.execute(query, execution_profile=tmp_profile)
queried_hosts.add(rs.response_future._current_host)
self.assertEqual(queried_hosts, expected_hosts)
tuple_row = rs[0]
self.assertIsInstance(tuple_row, tuple)
with self.assertRaises(AttributeError):
tuple_row.release_version
# make sure original profile is not impacted
self.assertTrue(
session.execute(query,
execution_profile='node1')[0].release_version)
def test_add_profile_timeout(self):
"""
Tests that EP Timeouts are honored.
@since 3.5
@jira_ticket PYTHON-569
@expected_result EP timeouts should override defaults
@test_category config_profiles
"""
node1 = ExecutionProfile(
load_balancing_policy=WhiteListRoundRobinPolicy(['127.0.0.1']))
with Cluster(
execution_profiles={EXEC_PROFILE_DEFAULT: node1}) as cluster:
session = cluster.connect(wait_for_all_pools=True)
pools = session.get_pool_state()
self.assertGreater(len(cluster.metadata.all_hosts()), 2)
self.assertEqual(set(h.address for h in pools), set(
('127.0.0.1', )))
node2 = ExecutionProfile(
load_balancing_policy=WhiteListRoundRobinPolicy(['127.0.0.2']))
max_retry_count = 10
for i in range(max_retry_count):
start = time.time()
try:
self.assertRaises(dse.OperationTimedOut,
cluster.add_execution_profile,
'profile_{0}'.format(i),
node2,
pool_wait_timeout=sys.float_info.min)
break
except AssertionError:
end = time.time()
self.assertAlmostEqual(start, end, 1)
break
else:
raise Exception(
"add_execution_profile didn't timeout after {0} retries".
format(max_retry_count))
class DuplicateRpcTest(unittest.TestCase):
load_balancing_policy = WhiteListRoundRobinPolicy(['127.0.0.1'])
def setUp(self):
self.cluster = Cluster(
protocol_version=PROTOCOL_VERSION,
execution_profiles={
EXEC_PROFILE_DEFAULT:
ExecutionProfile(
load_balancing_policy=self.load_balancing_policy)
})
self.session = self.cluster.connect()
self.session.execute(
"UPDATE system.peers SET rpc_address = '127.0.0.1' WHERE peer='127.0.0.2'"
)
def tearDown(self):
self.session.execute(
"UPDATE system.peers SET rpc_address = '127.0.0.2' WHERE peer='127.0.0.2'"
)
self.cluster.shutdown()
def test_duplicate(self):
"""
Test duplicate RPC addresses.
Modifies the system.peers table to make hosts have the same rpc address. Ensures such hosts are filtered out and a message is logged
@since 3.4
@jira_ticket PYTHON-366
@expected_result only one hosts' metadata will be populated
@test_category metadata
"""
mock_handler = MockLoggingHandler()
logger = logging.getLogger(dse.cluster.__name__)
logger.addHandler(mock_handler)
test_cluster = self.cluster = Cluster(
protocol_version=PROTOCOL_VERSION,
execution_profiles={
EXEC_PROFILE_DEFAULT:
ExecutionProfile(
load_balancing_policy=self.load_balancing_policy)
})
test_cluster.connect()
warnings = mock_handler.messages.get("warning")
self.assertEqual(len(warnings), 1)
self.assertTrue('multiple' in warnings[0])
logger.removeHandler(mock_handler)
test_cluster.shutdown()
def setUp(self):
contact_point = ['127.0.0.2']
self.cluster = Cluster(
contact_points=contact_point,
metrics_enabled=True,
protocol_version=PROTOCOL_VERSION,
execution_profiles={
EXEC_PROFILE_DEFAULT:
ExecutionProfile(
load_balancing_policy=WhiteListRoundRobinPolicy(
contact_point),
retry_policy=FallthroughRetryPolicy())
})
self.session = self.cluster.connect("test3rf", wait_for_all_pools=True)
def test_hosts_with_hostname(self):
hosts = ['localhost']
policy = WhiteListRoundRobinPolicy(hosts)
host = Host("127.0.0.1", SimpleConvictionPolicy)
policy.populate(None, [host])
qplan = list(policy.make_query_plan())
self.assertEqual(sorted(qplan), [host])
self.assertEqual(policy.distance(host), HostDistance.LOCAL)
def test_white_list(self):
use_singledc()
keyspace = 'test_white_list'
cluster = Cluster(
('127.0.0.2', ),
protocol_version=PROTOCOL_VERSION,
topology_event_refresh_window=0,
status_event_refresh_window=0,
execution_profiles={
EXEC_PROFILE_DEFAULT:
ExecutionProfile(
load_balancing_policy=WhiteListRoundRobinPolicy((
IP_FORMAT % 2, )))
})
session = cluster.connect()
self._wait_for_nodes_up([1, 2, 3])
create_schema(cluster, session, keyspace)
self._insert(session, keyspace)
self._query(session, keyspace)
self.coordinator_stats.assert_query_count_equals(self, 1, 0)
self.coordinator_stats.assert_query_count_equals(self, 2, 12)
self.coordinator_stats.assert_query_count_equals(self, 3, 0)
# white list policy should not allow reconnecting to ignored hosts
force_stop(3)
self._wait_for_nodes_down([3])
self.assertFalse(
cluster.metadata._hosts[IP_FORMAT % 3].is_currently_reconnecting())
self.coordinator_stats.reset_counts()
force_stop(2)
self._wait_for_nodes_down([2])
try:
self._query(session, keyspace)
self.fail()
except NoHostAvailable:
pass
finally:
cluster.shutdown()
def setUp(self):
"""
Setup sessions and pause node1
"""
self.cluster = Cluster(
protocol_version=PROTOCOL_VERSION,
execution_profiles={
EXEC_PROFILE_DEFAULT:
ExecutionProfile(
load_balancing_policy=WhiteListRoundRobinPolicy(
['127.0.0.1']))
})
self.session = self.cluster.connect()
ddl = '''
CREATE TABLE test3rf.timeout (
k int PRIMARY KEY,
v int )'''
self.session.execute(ddl)
self.node1 = get_node(1)
self.node1.pause()
def connect():
"""Connect to a Cassandra cluster.
keyspace, hosts, strategy variables are used to configure the connection.
See the cfg object in the :mod:`radon.model.config` module, .
:return: A boolean which indicates if the connection is successful
:rtype: bool"""
num_retries = 5
retry_timeout = 2
keyspace = cfg.dse_keyspace
hosts = cfg.dse_host
strategy = (cfg.dse_strategy, )
for _ in range(num_retries):
try:
cfg.logger.info('Connecting to Cassandra keyspace "{2}" '
'on "{0}" with strategy "{1}"'.format(
hosts, strategy, keyspace))
profile = ExecutionProfile(
load_balancing_policy=WhiteListRoundRobinPolicy(hosts))
profiles = {EXEC_PROFILE_DEFAULT: profile}
connection.setup(
hosts,
keyspace,
protocol_version=3,
execution_profiles=profiles,
)
return True
except NoHostAvailable:
cfg.logger.warning(
"Unable to connect to Cassandra on {0}. Retrying in {1} seconds..."
.format(hosts, retry_timeout))
time.sleep(retry_timeout)
return False
def test_refresh_schema_no_wait(self):
contact_points = [DSE_IP]
with Cluster(protocol_version=PROTOCOL_VERSION,
max_schema_agreement_wait=10,
contact_points=contact_points,
execution_profiles={
EXEC_PROFILE_DEFAULT:
ExecutionProfile(
load_balancing_policy=WhiteListRoundRobinPolicy(
contact_points))
}) as cluster:
session = cluster.connect()
schema_ver = session.execute(
"SELECT schema_version FROM system.local WHERE key='local'"
)[0][0]
new_schema_ver = uuid4()
session.execute(
"UPDATE system.local SET schema_version=%s WHERE key='local'",
(new_schema_ver, ))
try:
agreement_timeout = 1
# cluster agreement wait exceeded
c = Cluster(protocol_version=PROTOCOL_VERSION,
max_schema_agreement_wait=agreement_timeout)
c.connect()
self.assertTrue(c.metadata.keyspaces)
# cluster agreement wait used for refresh
original_meta = c.metadata.keyspaces
start_time = time.time()
self.assertRaisesRegexp(
Exception, r"Schema metadata was not refreshed.*",
c.refresh_schema_metadata)
end_time = time.time()
self.assertGreaterEqual(end_time - start_time,
agreement_timeout)
self.assertIs(original_meta, c.metadata.keyspaces)
# refresh wait overrides cluster value
original_meta = c.metadata.keyspaces
start_time = time.time()
c.refresh_schema_metadata(max_schema_agreement_wait=0)
end_time = time.time()
self.assertLess(end_time - start_time, agreement_timeout)
self.assertIsNot(original_meta, c.metadata.keyspaces)
self.assertEqual(original_meta, c.metadata.keyspaces)
c.shutdown()
refresh_threshold = 0.5
# cluster agreement bypass
c = Cluster(protocol_version=PROTOCOL_VERSION,
max_schema_agreement_wait=0)
start_time = time.time()
s = c.connect()
end_time = time.time()
self.assertLess(end_time - start_time, refresh_threshold)
self.assertTrue(c.metadata.keyspaces)
# cluster agreement wait used for refresh
original_meta = c.metadata.keyspaces
start_time = time.time()
c.refresh_schema_metadata()
end_time = time.time()
self.assertLess(end_time - start_time, refresh_threshold)
self.assertIsNot(original_meta, c.metadata.keyspaces)
self.assertEqual(original_meta, c.metadata.keyspaces)
# refresh wait overrides cluster value
original_meta = c.metadata.keyspaces
start_time = time.time()
self.assertRaisesRegexp(
Exception,
r"Schema metadata was not refreshed.*",
c.refresh_schema_metadata,
max_schema_agreement_wait=agreement_timeout)
end_time = time.time()
self.assertGreaterEqual(end_time - start_time,
agreement_timeout)
self.assertIs(original_meta, c.metadata.keyspaces)
c.shutdown()
finally:
# TODO once fixed this connect call
session = cluster.connect()
session.execute(
"UPDATE system.local SET schema_version=%s WHERE key='local'",
(schema_ver, ))
class ContextManagementTest(unittest.TestCase):
load_balancing_policy = WhiteListRoundRobinPolicy([DSE_IP])
cluster_kwargs = {
'execution_profiles': {
EXEC_PROFILE_DEFAULT:
ExecutionProfile(load_balancing_policy=load_balancing_policy)
},
'schema_metadata_enabled': False,
'token_metadata_enabled': False
}
def test_no_connect(self):
"""
Test cluster context without connecting.
@since 3.4
@jira_ticket PYTHON-521
@expected_result context should still be valid
@test_category configuration
"""
with Cluster() as cluster:
self.assertFalse(cluster.is_shutdown)
self.assertTrue(cluster.is_shutdown)
def test_simple_nested(self):
"""
Test cluster and session contexts nested in one another.
@since 3.4
@jira_ticket PYTHON-521
@expected_result cluster/session should be crated and shutdown appropriately.
@test_category configuration
"""
with Cluster(**self.cluster_kwargs) as cluster:
with cluster.connect() as session:
self.assertFalse(cluster.is_shutdown)
self.assertFalse(session.is_shutdown)
self.assertTrue(
session.execute('select release_version from system.local')
[0])
self.assertTrue(session.is_shutdown)
self.assertTrue(cluster.is_shutdown)
def test_cluster_no_session(self):
"""
Test cluster context without session context.
@since 3.4
@jira_ticket PYTHON-521
@expected_result Session should be created correctly. Cluster should shutdown outside of context
@test_category configuration
"""
with Cluster(**self.cluster_kwargs) as cluster:
session = cluster.connect()
self.assertFalse(cluster.is_shutdown)
self.assertFalse(session.is_shutdown)
self.assertTrue(
session.execute('select release_version from system.local')[0])
self.assertTrue(session.is_shutdown)
self.assertTrue(cluster.is_shutdown)
def test_session_no_cluster(self):
"""
Test session context without cluster context.
@since 3.4
@jira_ticket PYTHON-521
@expected_result session should be created correctly. Session should shutdown correctly outside of context
@test_category configuration
"""
cluster = Cluster(**self.cluster_kwargs)
unmanaged_session = cluster.connect()
with cluster.connect() as session:
self.assertFalse(cluster.is_shutdown)
self.assertFalse(session.is_shutdown)
self.assertFalse(unmanaged_session.is_shutdown)
self.assertTrue(
session.execute('select release_version from system.local')[0])
self.assertTrue(session.is_shutdown)
self.assertFalse(cluster.is_shutdown)
self.assertFalse(unmanaged_session.is_shutdown)
unmanaged_session.shutdown()
self.assertTrue(unmanaged_session.is_shutdown)
self.assertFalse(cluster.is_shutdown)
cluster.shutdown()
self.assertTrue(cluster.is_shutdown)
def __init__(self,
hostname,
port,
color=False,
username=None,
password=None,
encoding=None,
stdin=None,
tty=True,
completekey=cqlsh.DEFAULT_COMPLETEKEY,
browser=None,
use_conn=None,
cqlver=None,
keyspace=None,
tracing_enabled=False,
expand_enabled=False,
no_compact=False,
display_nanotime_format=cqlsh.DEFAULT_NANOTIME_FORMAT,
display_timestamp_format=cqlsh.DEFAULT_TIMESTAMP_FORMAT,
display_date_format=cqlsh.DEFAULT_DATE_FORMAT,
display_float_precision=cqlsh.DEFAULT_FLOAT_PRECISION,
display_double_precision=cqlsh.DEFAULT_DOUBLE_PRECISION,
display_timezone=None,
max_trace_wait=cqlsh.DEFAULT_MAX_TRACE_WAIT,
ssl=False,
single_statement=None,
request_timeout=cqlsh.DEFAULT_REQUEST_TIMEOUT_SECONDS,
protocol_version=None,
connect_timeout=cqlsh.DEFAULT_CONNECT_TIMEOUT_SECONDS,
no_file_io=cqlsh.DEFAULT_NO_FILE_IO,
is_subshell=False):
from dselib.authfactory import get_auth_provider
if username:
if not password:
password = getpass.getpass()
auth_provider = get_auth_provider(cqlsh.CONFIG_FILE, os.environ,
username, password, cqlsh.options)
self.execute_as = None
self.consistency_level = dse.ConsistencyLevel.ONE
self.serial_consistency_level = dse.ConsistencyLevel.SERIAL
conn = None
execution_profile = ExecutionProfile(
load_balancing_policy=WhiteListRoundRobinPolicy([hostname]),
row_factory=ordered_dict_factory,
request_timeout=request_timeout,
consistency_level=self.consistency_level,
serial_consistency_level=self.serial_consistency_level)
self.execution_profiles = {EXEC_PROFILE_DEFAULT: execution_profile}
if use_conn:
conn = use_conn
else:
kwargs = {}
if protocol_version is not None:
kwargs['protocol_version'] = protocol_version
conn = cqlsh.Cluster(
contact_points=(hostname, ),
port=port,
cql_version=cqlver,
auth_provider=auth_provider,
ssl_options=sslhandling.ssl_settings(
hostname, cqlsh.CONFIG_FILE) if ssl else None,
control_connection_timeout=connect_timeout,
connect_timeout=connect_timeout,
execution_profiles=self.execution_profiles,
**kwargs)
OriginalShell.__init__(
self,
hostname,
port,
color=color,
auth_provider=auth_provider,
username=username,
password=password,
encoding=encoding,
stdin=stdin,
tty=tty,
completekey=completekey,
browser=browser,
use_conn=conn,
cqlver=cqlver,
keyspace=keyspace,
tracing_enabled=tracing_enabled,
expand_enabled=expand_enabled,
no_compact=no_compact,
display_nanotime_format=display_nanotime_format,
display_timestamp_format=display_timestamp_format,
display_date_format=display_date_format,
display_float_precision=display_float_precision,
display_double_precision=display_double_precision,
display_timezone=display_timezone,
max_trace_wait=max_trace_wait,
ssl=ssl,
single_statement=single_statement,
request_timeout=request_timeout,
protocol_version=protocol_version,
connect_timeout=connect_timeout,
no_file_io=no_file_io,
is_subshell=is_subshell)
self.owns_connection = not use_conn
self.execution_profile = self.session.get_execution_profile(
EXEC_PROFILE_DEFAULT)