def test_write_timeout(self): policy = FallthroughRetryPolicy() retry, consistency = policy.on_write_timeout( query=None, consistency="ONE", write_type=WriteType.SIMPLE, required_responses=1, received_responses=2, retry_num=1) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) retry, consistency = policy.on_write_timeout( query=None, consistency="ONE", write_type=WriteType.SIMPLE, required_responses=1, received_responses=2, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) retry, consistency = policy.on_write_timeout( query=None, consistency="ONE", write_type=WriteType.BATCH_LOG, required_responses=10000, received_responses=1, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None)
def test_unavailable(self): policy = FallthroughRetryPolicy() retry, consistency = policy.on_unavailable(query=None, consistency="ONE", required_replicas=1, alive_replicas=2, retry_num=1) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) retry, consistency = policy.on_unavailable(query=None, consistency="ONE", required_replicas=1, alive_replicas=2, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) retry, consistency = policy.on_unavailable(query=None, consistency="ONE", required_replicas=10000, alive_replicas=1, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None)
def test_read_timeout(self): policy = FallthroughRetryPolicy() retry, consistency = policy.on_read_timeout( query=None, consistency=ONE, required_responses=1, received_responses=2, data_retrieved=True, retry_num=1) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) retry, consistency = policy.on_read_timeout( query=None, consistency=ONE, required_responses=2, received_responses=1, data_retrieved=True, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) retry, consistency = policy.on_read_timeout( query=None, consistency=ONE, required_responses=2, received_responses=2, data_retrieved=True, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) retry, consistency = policy.on_read_timeout( query=None, consistency=ONE, required_responses=2, received_responses=2, data_retrieved=False, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None)
def test_write_timeout(self): policy = FallthroughRetryPolicy() retry, consistency = policy.on_write_timeout( query=None, consistency=ONE, write_type=WriteType.SIMPLE, required_responses=1, received_responses=2, retry_num=1, ) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) retry, consistency = policy.on_write_timeout( query=None, consistency=ONE, write_type=WriteType.SIMPLE, required_responses=1, received_responses=2, retry_num=0, ) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) retry, consistency = policy.on_write_timeout( query=None, consistency=ONE, write_type=WriteType.BATCH_LOG, required_responses=10000, received_responses=1, retry_num=0, ) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None)
def test_duplicate_metrics_per_cluster(self): """ Test to validate that cluster metrics names can't overlap. @since 3.6.0 @jira_ticket PYTHON-561 @expected_result metric names should not be allowed to be same. @test_category metrics """ cluster2 = Cluster(metrics_enabled=True, protocol_version=PROTOCOL_VERSION, default_retry_policy=FallthroughRetryPolicy()) cluster3 = Cluster(metrics_enabled=True, protocol_version=PROTOCOL_VERSION, default_retry_policy=FallthroughRetryPolicy()) # Ensure duplicate metric names are not allowed cluster2.metrics.set_stats_name("appcluster") cluster2.metrics.set_stats_name("appcluster") with self.assertRaises(ValueError): cluster3.metrics.set_stats_name("appcluster") cluster3.metrics.set_stats_name("devops") session2 = cluster2.connect(self.ks_name, wait_for_all_pools=True) session3 = cluster3.connect(self.ks_name, wait_for_all_pools=True) # Basic validation that naming metrics doesn't impact their segration or accuracy for i in range(10): query = SimpleStatement("SELECT * FROM {0}.{0}".format( self.ks_name), consistency_level=ConsistencyLevel.ALL) session2.execute(query) for i in range(5): query = SimpleStatement("SELECT * FROM {0}.{0}".format( self.ks_name), consistency_level=ConsistencyLevel.ALL) session3.execute(query) self.assertEqual( cluster2.metrics.get_stats()['request_timer']['count'], 10) self.assertEqual( cluster3.metrics.get_stats()['request_timer']['count'], 5) # Check scales to ensure they are appropriately named self.assertTrue("appcluster" in scales._Stats.stats.keys()) self.assertTrue("devops" in scales._Stats.stats.keys()) cluster2.shutdown() cluster3.shutdown()
def __init__(self, log_level=logging.INFO): """ Constructor. """ class_name = self.__class__.__name__ self.logger = logging.getLogger(class_name) self.logger.setLevel(log_level) self.logger.info('Starting {}'.format(class_name)) self.hosts = appscale_info.get_db_ips() self.retry_policy = IdempotentRetryPolicy() self.no_retries = FallthroughRetryPolicy() remaining_retries = INITIAL_CONNECT_RETRIES while True: try: self.cluster = Cluster(self.hosts, default_retry_policy=self.retry_policy) self.session = self.cluster.connect(KEYSPACE) break except cassandra.cluster.NoHostAvailable as connection_error: remaining_retries -= 1 if remaining_retries < 0: raise connection_error time.sleep(3) self.session.default_consistency_level = ConsistencyLevel.QUORUM
def index_query_test(self): """ Check that a secondary index query times out """ cluster = self.cluster cluster.set_configuration_options(values={'read_request_timeout_in_ms': 1000}) cluster.populate(1).start(wait_for_binary_proto=True, jvm_args=["-Dcassandra.monitoring_check_interval_ms=50", "-Dcassandra.test.read_iteration_delay_ms=1500"]) # see above for explanation node = cluster.nodelist()[0] session = self.patient_cql_connection(node) self.create_ks(session, 'ks', 1) session.execute(""" CREATE TABLE test3 ( id int PRIMARY KEY, col int, val text ); """) session.execute("CREATE INDEX ON test3 (col)") for i in xrange(500): session.execute("INSERT INTO test3 (id, col, val) VALUES ({}, {}, 'foo')".format(i, i // 10)) mark = node.mark_log() statement = session.prepare("SELECT * from test3 WHERE col < ? ALLOW FILTERING") statement.consistency_level = ConsistencyLevel.ONE statement.retry_policy = FallthroughRetryPolicy() assert_unavailable(lambda c: debug(c.execute(statement, [50])), session) node.watch_log_for("Some operations timed out", from_mark=mark, timeout=60)
def test_metrics_per_cluster(self): """ Test to validate that metrics can be scopped to invdividual clusters @since 3.6.0 @jira_ticket PYTHON-561 @expected_result metrics should be scopped to a cluster level @test_category metrics """ cluster2 = Cluster(metrics_enabled=True, protocol_version=PROTOCOL_VERSION, default_retry_policy=FallthroughRetryPolicy()) cluster2.connect(self.ks_name, wait_for_all_pools=True) self.assertEqual(len(cluster2.metadata.all_hosts()), 3) query = SimpleStatement("SELECT * FROM {0}.{0}".format(self.ks_name), consistency_level=ConsistencyLevel.ALL) self.session.execute(query) # Pause node so it shows as unreachable to coordinator get_node(1).pause() try: # Test write query = SimpleStatement( "INSERT INTO {0}.{0} (k, v) VALUES (2, 2)".format( self.ks_name), consistency_level=ConsistencyLevel.ALL) with self.assertRaises(WriteTimeout): self.session.execute(query, timeout=None) finally: get_node(1).resume() # Change the scales stats_name of the cluster2 cluster2.metrics.set_stats_name('cluster2-metrics') stats_cluster1 = self.cluster.metrics.get_stats() stats_cluster2 = cluster2.metrics.get_stats() # Test direct access to stats self.assertEqual(1, self.cluster.metrics.stats.write_timeouts) self.assertEqual(0, cluster2.metrics.stats.write_timeouts) # Test direct access to a child stats self.assertNotEqual(0.0, self.cluster.metrics.request_timer['mean']) self.assertEqual(0.0, cluster2.metrics.request_timer['mean']) # Test access via metrics.get_stats() self.assertNotEqual(0.0, stats_cluster1['request_timer']['mean']) self.assertEqual(0.0, stats_cluster2['request_timer']['mean']) # Test access by stats_name self.assertEqual( 0.0, scales.getStats()['cluster2-metrics']['request_timer']['mean']) cluster2.shutdown()
def _insert_tombstones(self, session, number_of_tombstones): for num_id in range(number_of_tombstones): session.execute( SimpleStatement( "DELETE value FROM tombstonefailure WHERE id = {}".format( num_id), consistency_level=self.consistency_level, retry_policy=FallthroughRetryPolicy()))
def setUp(self): contact_point = ['127.0.0.2'] self.cluster = Cluster(contact_points=contact_point, metrics_enabled=True, protocol_version=PROTOCOL_VERSION, load_balancing_policy=HostFilterPolicy( RoundRobinPolicy(), lambda host: host.address in contact_point ), default_retry_policy=FallthroughRetryPolicy()) self.session = self.cluster.connect("test3rf", wait_for_all_pools=True)
def create(self, location): """ Saves a location in the database Args: location: model.Location to insert into database """ query = """ INSERT INTO location ( username, latitude, longitude, timestamp_created, location_name ) VALUES ( %(username)s, %(latitude)s, %(longitude)s, %(timestamp_created)s, %(location_name)s ) """ location_dict = location.to_dict() client = current_app.extensions['registry']['CASSANDRA_CLIENT'] client.execute(query, params=location_dict, routing_key=location.username, retry_policy=FallthroughRetryPolicy()) query = """ INSERT INTO location_by_timestamp ( username, latitude, longitude, timestamp_created, location_name ) VALUES ( %(username)s, %(latitude)s, %(longitude)s, %(timestamp_created)s, %(location_name)s ) """ client.execute(query, params=location_dict, routing_key=location.username, retry_policy=FallthroughRetryPolicy())
def materialized_view_test(self): """ Check that a materialized view query times out: - populate a 2-node cluster - set a 1-second read timeout - start one node without having it join the ring - start the other node with read_iteration_delay set to 1.5 seconds - (this will cause read queries to take longer than the read timeout) - CREATE a table - CREATE a materialized view over that table - INSERT 50 values into that table - assert querying that table results in an unavailable exception """ cluster = self.cluster cluster.set_configuration_options( values={'read_request_timeout_in_ms': 1000}) cluster.populate(2) node1, node2 = cluster.nodelist() node1.start(wait_for_binary_proto=True, join_ring=False) # ensure other node executes queries node2.start(wait_for_binary_proto=True, jvm_args=[ "-Dcassandra.monitoring_check_interval_ms=50", "-Dcassandra.test.read_iteration_delay_ms=1500" ]) # see above for explanation session = self.patient_exclusive_cql_connection(node1) self.create_ks(session, 'ks', 1) session.execute(""" CREATE TABLE test4 ( id int PRIMARY KEY, col int, val text ); """) session.execute( ("CREATE MATERIALIZED VIEW mv AS SELECT * FROM test4 " "WHERE col IS NOT NULL AND id IS NOT NULL PRIMARY KEY (col, id)")) for i in range(50): session.execute( "INSERT INTO test4 (id, col, val) VALUES ({}, {}, 'foo')". format(i, i // 10)) mark = node2.mark_log() statement = SimpleStatement("SELECT * FROM mv WHERE col = 50", consistency_level=ConsistencyLevel.ONE, retry_policy=FallthroughRetryPolicy()) assert_unavailable(lambda c: debug(c.execute(statement)), session) node2.watch_log_for("operations timed out", from_mark=mark, timeout=60)
def test_unavailable(self): policy = FallthroughRetryPolicy() retry, consistency = policy.on_unavailable( query=None, consistency=ONE, required_replicas=1, alive_replicas=2, retry_num=1) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) retry, consistency = policy.on_unavailable( query=None, consistency=ONE, required_replicas=1, alive_replicas=2, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) retry, consistency = policy.on_unavailable( query=None, consistency=ONE, required_replicas=10000, alive_replicas=1, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None)
def test_read_timeout(self): policy = FallthroughRetryPolicy() retry, consistency = policy.on_read_timeout( query=None, consistency="ONE", required_responses=1, received_responses=2, data_retrieved=True, retry_num=1) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) retry, consistency = policy.on_read_timeout( query=None, consistency="ONE", required_responses=2, received_responses=1, data_retrieved=True, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) retry, consistency = policy.on_read_timeout( query=None, consistency="ONE", required_responses=2, received_responses=2, data_retrieved=True, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None) retry, consistency = policy.on_read_timeout( query=None, consistency="ONE", required_responses=2, received_responses=2, data_retrieved=False, retry_num=0) self.assertEqual(retry, RetryPolicy.RETHROW) self.assertEqual(consistency, None)
def _perform_cql_statement(self, session, text_statement): statement = SimpleStatement(text_statement, consistency_level=self.consistency_level, retry_policy=FallthroughRetryPolicy()) if self.expected_expt is None: session.execute(statement) else: with pytest.raises(self.expected_expt) as cm: # On 2.1, we won't return the ReadTimeout from coordinator until actual timeout, # so we need to up the default timeout of the driver session session.execute(statement, timeout=15) return cm._excinfo[1]
def setUp(self): contact_point = ['127.0.0.2'] self.cluster = TestCluster(contact_points=contact_point, metrics_enabled=True, execution_profiles= {EXEC_PROFILE_DEFAULT: ExecutionProfile( load_balancing_policy=HostFilterPolicy( RoundRobinPolicy(), lambda host: host.address in contact_point), retry_policy=FallthroughRetryPolicy() ) } ) self.session = self.cluster.connect("test3rf", wait_for_all_pools=True)
def local_query_test(self): """ Check that a query running on the local coordinator node times out: - set a 1-second read timeout - start the cluster with read_iteration_delay set to 1.5 seconds - (this will cause read queries to take longer than the read timeout) - CREATE and INSERT into a table - SELECT * from the table using a retry policy that never retries, and assert it times out @jira_ticket CASSANDRA-7392 """ cluster = self.cluster cluster.set_configuration_options( values={'read_request_timeout_in_ms': 1000}) # cassandra.test.read_iteration_delay_ms causes the state tracking read iterators # introduced by CASSANDRA-7392 to pause by the specified amount of milliseconds during each # iteration of non system queries, so that these queries take much longer to complete, # see ReadCommand.withStateTracking() cluster.populate(1).start( wait_for_binary_proto=True, jvm_args=[ "-Dcassandra.monitoring_check_interval_ms=50", "-Dcassandra.test.read_iteration_delay_ms=1500" ]) node = cluster.nodelist()[0] session = self.patient_cql_connection(node) self.create_ks(session, 'ks', 1) session.execute(""" CREATE TABLE test1 ( id int PRIMARY KEY, val text ); """) for i in range(500): session.execute( "INSERT INTO test1 (id, val) VALUES ({}, 'foo')".format(i)) mark = node.mark_log() statement = SimpleStatement("SELECT * from test1", consistency_level=ConsistencyLevel.ONE, retry_policy=FallthroughRetryPolicy()) assert_unavailable(lambda c: debug(c.execute(statement)), session) node.watch_log_for("operations timed out", from_mark=mark, timeout=60)
def setup_once(self): cluster = self.cluster cluster.set_configuration_options({ 'read_request_timeout_in_ms': 3000, 'write_request_timeout_in_ms': 3000, 'phi_convict_threshold': 12, 'tombstone_failure_threshold': TOMBSTONE_FAILURE_THRESHOLD, 'enable_materialized_views': 'true' }) cluster.populate(2, debug=True) cluster.start(jvm_args=JVM_ARGS) node1 = cluster.nodelist()[0] global jmx jmx = JolokiaAgent(node1) jmx.start() s = self.session = self.patient_exclusive_cql_connection( node1, retry_policy=FallthroughRetryPolicy(), request_timeout=30) for k in [KEYSPACE, FAIL_WRITE_KEYSPACE]: create_ks(s, k, 2) s.execute( f"CREATE TABLE {k}.{TABLE} (k int, c int, v int, PRIMARY KEY (k,c))" ) create_ks(s, VIEW_KEYSPACE, 1) s.execute( f"CREATE TABLE {VIEW_KEYSPACE}.{TABLE} (k int, c int, v int, PRIMARY KEY (k,c))" ) s.execute( f"CREATE MATERIALIZED VIEW {VIEW_KEYSPACE}.{VIEW} AS SELECT * FROM {TABLE} WHERE c IS NOT NULL AND k IS NOT NULL PRIMARY KEY (c,k);" ) # Here we're doing a series of deletions in order to create enough tombstones to exceed the configured fail threshold. # This partition will be used to test read failures. for c in range(TOMBSTONE_FAILURE_THRESHOLD + 1): self.session.execute( f"DELETE FROM {KEYSPACE}.{TABLE} WHERE k={TOMBSTONE_FAIL_KEY} AND c={c}" ) node1.watch_log_for( "Created default superuser role 'cassandra'" ) # don't race with async default role creation, which creates a write node1.watch_log_for( 'Completed submission of build tasks for any materialized views defined at startup', filename='debug.log') # view builds cause background reads
def remote_query_test(self): """ Check that a query running on a node other than the coordinator times out: - populate the cluster with 2 nodes - set a 1-second read timeout - start one node without having it join the ring - start the other node with read_iteration_delay set to 1.5 seconds - (this will cause read queries to take longer than the read timeout) - CREATE a table - INSERT 5000 rows on a session on the node that is not a member of the ring - run SELECT statements and assert they fail # TODO refactor SELECT statements: # - run the statements in a loop to reduce duplication # - watch the log after each query # - assert we raise the right error """ cluster = self.cluster cluster.set_configuration_options( values={'read_request_timeout_in_ms': 1000}) cluster.populate(2) node1, node2 = cluster.nodelist() node1.start(wait_for_binary_proto=True, join_ring=False) # ensure other node executes queries node2.start(wait_for_binary_proto=True, jvm_args=[ "-Dcassandra.monitoring_check_interval_ms=50", "-Dcassandra.test.read_iteration_delay_ms=1500" ]) # see above for explanation session = self.patient_exclusive_cql_connection(node1) self.create_ks(session, 'ks', 1) session.execute(""" CREATE TABLE test2 ( id int, col int, val text, PRIMARY KEY(id, col) ); """) for i, j in itertools.product(range(500), range(10)): session.execute( "INSERT INTO test2 (id, col, val) VALUES ({}, {}, 'foo')". format(i, j)) mark = node2.mark_log() statement = SimpleStatement("SELECT * from test2", consistency_level=ConsistencyLevel.ONE, retry_policy=FallthroughRetryPolicy()) assert_unavailable(lambda c: debug(c.execute(statement)), session) statement = SimpleStatement("SELECT * from test2 where id = 1", consistency_level=ConsistencyLevel.ONE, retry_policy=FallthroughRetryPolicy()) assert_unavailable(lambda c: debug(c.execute(statement)), session) statement = SimpleStatement( "SELECT * from test2 where id IN (1, 10, 20) AND col < 10", consistency_level=ConsistencyLevel.ONE, retry_policy=FallthroughRetryPolicy()) assert_unavailable(lambda c: debug(c.execute(statement)), session) statement = SimpleStatement( "SELECT * from test2 where col > 5 ALLOW FILTERING", consistency_level=ConsistencyLevel.ONE, retry_policy=FallthroughRetryPolicy()) assert_unavailable(lambda c: debug(c.execute(statement)), session) node2.watch_log_for("operations timed out", from_mark=mark, timeout=60)
from cassandra.cluster import SimpleStatement from cassandra.policies import FallthroughRetryPolicy, RetryPolicy from .cassandra_interface import IndexStates from .cassandra_interface import INITIAL_CONNECT_RETRIES from .cassandra_interface import KEYSPACE from .cassandra_interface import ScatterPropStates from .cassandra_interface import ThriftColumn from .constants import CURRENT_VERSION, LB_POLICY from .. import dbconstants sys.path.append(APPSCALE_PYTHON_APPSERVER) from google.appengine.datastore import entity_pb from google.net.proto.ProtocolBuffer import ProtocolBufferDecodeError # A policy that does not retry statements. NO_RETRIES = FallthroughRetryPolicy() logger = logging.getLogger(__name__) # The number of times to retry idempotent statements. BASIC_RETRY_COUNT = 5 class IdempotentRetryPolicy(RetryPolicy): """ A policy used for retrying idempotent statements. """ def on_read_timeout(self, query, consistency, required_responses, received_responses, data_retrieved, retry_num): """ This is called when a ReadTimeout occurs. Args: query: A statement that timed out.
def remote_query_test(self): """ Check that a query running on a node other than the coordinator times out """ cluster = self.cluster cluster.set_configuration_options( values={'read_request_timeout_in_ms': 1000}) cluster.populate(2) node1, node2 = cluster.nodelist() node1.start(wait_for_binary_proto=True, join_ring=False) # ensure other node executes queries node2.start(wait_for_binary_proto=True, jvm_args=[ "-Dcassandra.monitoring_check_interval_ms=50", "-Dcassandra.test.read_iteration_delay_ms=1500" ]) # see above for explanation session = self.patient_exclusive_cql_connection(node1) self.create_ks(session, 'ks', 1) session.execute(""" CREATE TABLE test2 ( id int, col int, val text, PRIMARY KEY(id, col) ); """) for i in xrange(500): for j in xrange(10): session.execute( "INSERT INTO test2 (id, col, val) VALUES ({}, {}, 'foo')". format(i, j)) mark = node2.mark_log() statement = SimpleStatement("SELECT * from test2", consistency_level=ConsistencyLevel.ONE, retry_policy=FallthroughRetryPolicy()) assert_unavailable(lambda c: debug(c.execute(statement)), session) statement = SimpleStatement("SELECT * from test2 where id = 1", consistency_level=ConsistencyLevel.ONE, retry_policy=FallthroughRetryPolicy()) assert_unavailable(lambda c: debug(c.execute(statement)), session) statement = SimpleStatement( "SELECT * from test2 where id IN (1, 10, 20) AND col < 10", consistency_level=ConsistencyLevel.ONE, retry_policy=FallthroughRetryPolicy()) assert_unavailable(lambda c: debug(c.execute(statement)), session) statement = SimpleStatement( "SELECT * from test2 where col > 5 ALLOW FILTERING", consistency_level=ConsistencyLevel.ONE, retry_policy=FallthroughRetryPolicy()) assert_unavailable(lambda c: debug(c.execute(statement)), session) node2.watch_log_for("Some operations timed out", from_mark=mark, timeout=60)