print("\nperformed %d actions in %s mode\n write actions: %d\n read actions: %d" % (actions, args.mode, write_actions, read_actions)) sys.exit(0) signal.signal(signal.SIGINT, sigint_handler) session = setup_session(args.cluster_ip, args.keyspace) if not args.keyspace: args.keyspace = select_random_keyspace(session) if random_table: args.table = select_random_table(session, args.keyspace) session.set_keyspace(args.keyspace) print("Selected table %s from keyspace %s" % (args.table, args.keyspace)) print("stressing database by sending {} queries every {} seconds...".format(args.mode, args.pause)) fetchStatement = SimpleStatement("SELECT * FROM " + args.table) rows = session.execute(fetchStatement) column_names = session.cluster.metadata.keyspaces[args.keyspace].tables[args.table].columns.keys() if args.mode == "READ": while True: read_from_table(session, args.table) actions += 1 pause(args.pause) if args.mode == "INSERT": rows = session.execute(fetchStatement) while True: insert_random_rows(session, args.table, rows.current_rows, column_names, args.batch_size) actions += 1 write_actions += 1 pause(args.pause)
def counter_consistency_test(self): """ Do a bunch of writes with ONE, read back with ALL and check results. """ cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() session = self.patient_cql_connection(node1) create_ks(session, 'counter_tests', 3) stmt = """ CREATE TABLE counter_table ( id uuid PRIMARY KEY, counter_one COUNTER, counter_two COUNTER, ) """ session.execute(stmt) counters = [] # establish 50 counters (2x25 rows) for i in xrange(25): _id = str(uuid.uuid4()) counters.append({_id: {'counter_one': 1, 'counter_two': 1}}) query = SimpleStatement(""" UPDATE counter_table SET counter_one = counter_one + 1, counter_two = counter_two + 1 where id = {uuid}""".format(uuid=_id), consistency_level=ConsistencyLevel.ONE) session.execute(query) # increment a bunch of counters with CL.ONE for i in xrange(10000): counter = counters[random.randint(0, len(counters) - 1)] counter_id = counter.keys()[0] query = SimpleStatement(""" UPDATE counter_table SET counter_one = counter_one + 2 where id = {uuid}""".format(uuid=counter_id), consistency_level=ConsistencyLevel.ONE) session.execute(query) query = SimpleStatement(""" UPDATE counter_table SET counter_two = counter_two + 10 where id = {uuid}""".format(uuid=counter_id), consistency_level=ConsistencyLevel.ONE) session.execute(query) query = SimpleStatement(""" UPDATE counter_table SET counter_one = counter_one - 1 where id = {uuid}""".format(uuid=counter_id), consistency_level=ConsistencyLevel.ONE) session.execute(query) query = SimpleStatement(""" UPDATE counter_table SET counter_two = counter_two - 5 where id = {uuid}""".format(uuid=counter_id), consistency_level=ConsistencyLevel.ONE) session.execute(query) # update expectations to match (assumed) db state counter[counter_id]['counter_one'] += 1 counter[counter_id]['counter_two'] += 5 # let's verify the counts are correct, using CL.ALL for counter_dict in counters: counter_id = counter_dict.keys()[0] query = SimpleStatement(""" SELECT counter_one, counter_two FROM counter_table WHERE id = {uuid} """.format(uuid=counter_id), consistency_level=ConsistencyLevel.ALL) rows = list(session.execute(query)) counter_one_actual, counter_two_actual = rows[0] self.assertEqual(counter_one_actual, counter_dict[counter_id]['counter_one']) self.assertEqual(counter_two_actual, counter_dict[counter_id]['counter_two'])
cluster = Cluster( seeds, load_balancing_policy=WhiteListRoundRobinPolicy(seeds), control_connection_timeout=180, connect_timeout=180 ) session = cluster.connect() session.row_factory = dict_factory session.default_fetch_size = 100 session.default_timeout = 180 count = 0 for key in keys: #print key[0], key[1] stat = "select version from aosmd_us_central_1.object_versions where bucket = '%s' and key = '%s';" % (key[0], key[1]) query = SimpleStatement(stat, consistency_level=ConsistencyLevel.LOCAL_QUORUM) res = session.execute(query) count = 0 for r in res: count += 1 try: version = r['version'] stat1 = "select * from aosmd_us_central_1.object_versions where bucket = '%s' and key = '%s' and version = '%s';" % (key[0], key[1], version) print stat1 query1 = SimpleStatement(stat, consistency_level=ConsistencyLevel.LOCAL_QUORUM) ret = session.execute(query1) except Exception, ex: print '%s failed' % version else: print '%s success' % version print count
def tombstone_failure_threshold_message_test(self): """ Ensure nodes return an error message in case of TombstoneOverwhelmingExceptions rather than dropping the request. A drop makes the coordinator waits for the specified read_request_timeout_in_ms. @jira_ticket CASSANDRA-7886 """ self.allow_log_errors = True self.cluster.set_configuration_options( values={ 'tombstone_failure_threshold': 500, 'read_request_timeout_in_ms': 30000, # 30 seconds 'range_request_timeout_in_ms': 40000 }) self.cluster.populate(3).start() node1, node2, node3 = self.cluster.nodelist() session = self.patient_cql_connection(node1) self.create_ks(session, 'test', 3) session.execute("CREATE TABLE test ( " "id int, mytext text, col1 int, col2 int, col3 int, " "PRIMARY KEY (id, mytext) )") # Add data with tombstones values = map(lambda i: str(i), range(1000)) for value in values: session.execute( SimpleStatement( "insert into test (id, mytext, col1) values (1, '{}', null) " .format(value), consistency_level=CL.ALL)) failure_msg = ("Scanned over.* tombstones.* query aborted") @timed(25) def read_failure_query(): assert_invalid(session, SimpleStatement( "select * from test where id in (1,2,3,4,5)", consistency_level=CL.ALL), expected=ReadFailure) read_failure_query() failure = (node1.grep_log(failure_msg) or node2.grep_log(failure_msg) or node3.grep_log(failure_msg)) self.assertTrue( failure, ("Cannot find tombstone failure threshold error in log " "after failed query")) mark1 = node1.mark_log() mark2 = node2.mark_log() mark3 = node3.mark_log() @timed(35) def range_request_failure_query(): assert_invalid(session, SimpleStatement("select * from test", consistency_level=CL.ALL), expected=ReadFailure) range_request_failure_query() failure = ( node1.watch_log_for(failure_msg, from_mark=mark1, timeout=5) or node2.watch_log_for(failure_msg, from_mark=mark2, timeout=5) or node3.watch_log_for(failure_msg, from_mark=mark3, timeout=5)) self.assertTrue( failure, ("Cannot find tombstone failure threshold error in log " "after range_request_timeout_query"))
def range_request_failure_query(): assert_invalid(session, SimpleStatement("select * from test", consistency_level=CL.ALL), expected=ReadFailure)
from cassandra import ConsistencyLevel from cassandra.cluster import Cluster from cassandra.query import SimpleStatement transaction2 = SimpleStatement( "BEGIN BATCH INSERT INTO vacancybot.users(first_name, last_name, balance, living_wage, birthday, city, skills) VALUES('Ioan', 'Petrov', 1800, 5000,toTimeStamp(toDate('2019-09-29 00:00:00+0000')), 'Kiev', [{name:'potions'}]) UPDATE vacancybot.vacancy SET salary = 24000 WHERE profession = 'Dancing teacher' AND name = 'Dancing teacher' and created_at = toTimeStamp(toDate('2019-09-29 00:00:00+0000')) APPLY BATCH", consistency_level=ConsistencyLevel.ANY) cluster = Cluster() session = cluster.connect('vacancybot') session.execute(transaction2)
def test_metrics_per_cluster(self): """ Test to validate that metrics can be scopped to invdividual clusters @since 3.6.0 @jira_ticket PYTHON-561 @expected_result metrics should be scopped to a cluster level @test_category metrics """ cluster2 = Cluster( metrics_enabled=True, protocol_version=PROTOCOL_VERSION, execution_profiles={ EXEC_PROFILE_DEFAULT: ExecutionProfile(retry_policy=FallthroughRetryPolicy()) }) cluster2.connect(self.ks_name, wait_for_all_pools=True) self.assertEqual(len(cluster2.metadata.all_hosts()), 3) query = SimpleStatement("SELECT * FROM {0}.{0}".format(self.ks_name), consistency_level=ConsistencyLevel.ALL) self.session.execute(query) # Pause node so it shows as unreachable to coordinator get_node(1).pause() try: # Test write query = SimpleStatement( "INSERT INTO {0}.{0} (k, v) VALUES (2, 2)".format( self.ks_name), consistency_level=ConsistencyLevel.ALL) with self.assertRaises(WriteTimeout): self.session.execute(query, timeout=None) finally: get_node(1).resume() # Change the scales stats_name of the cluster2 cluster2.metrics.set_stats_name('cluster2-metrics') stats_cluster1 = self.cluster.metrics.get_stats() stats_cluster2 = cluster2.metrics.get_stats() # Test direct access to stats self.assertEqual(1, self.cluster.metrics.stats.write_timeouts) self.assertEqual(0, cluster2.metrics.stats.write_timeouts) # Test direct access to a child stats self.assertNotEqual(0.0, self.cluster.metrics.request_timer['mean']) self.assertEqual(0.0, cluster2.metrics.request_timer['mean']) # Test access via metrics.get_stats() self.assertNotEqual(0.0, stats_cluster1['request_timer']['mean']) self.assertEqual(0.0, stats_cluster2['request_timer']['mean']) # Test access by stats_name self.assertEqual( 0.0, scales.getStats()['cluster2-metrics']['request_timer']['mean']) cluster2.shutdown()
def execute_async( self, query, params = {} ): if type( query ) is str or type( query ) is unicode: query = SimpleStatement( query, consistency_level = self.consistency ) return self.cur.execute_async( query, params )
# Parsing args hostname = sys.argv[1] #username = sys.argv[2] #password = sys.argv[3] desired_response_counter = int(sys.argv[2]) # Logging in #auth_provider = PlainTextAuthProvider(username=username, password=password) cluster = Cluster( [hostname], # auth_provider=auth_provider, load_balancing_policy=DCAwareRoundRobinPolicy(local_dc='us-east')) session = cluster.connect('replicated') # Predefining queries truncate_test = SimpleStatement("TRUNCATE replicated.test", consistency_level=ConsistencyLevel.ALL) truncate_test_count = SimpleStatement("TRUNCATE replicated.test_count", consistency_level=ConsistencyLevel.ALL) insert_into_test = SimpleStatement( """ INSERT INTO replicated.test (id, insertion_date, desired_response_counter, some_data) VALUES (%s, now(), %s, 'dummy'); """, consistency_level=ConsistencyLevel.LOCAL_ONE) while True: # Inserting new test value next_uuid = uuid.uuid1() response_counter = 0 session.execute(insert_into_test, (next_uuid, desired_response_counter)) # Waiting for all responses to arrive
def get_event_by_device0(args): session, _uuid, hour, msg = args _source, _kinds = get_source_kinds(msg.get("source"), msg.get("kinds")) # log("get_source_kinds {} {}".format(_source, _kinds)) if _source is None or _kinds is None: return "bad request: source {}, kinds {}".format(_source, _kinds) _end_m = parse_date(msg.get("end_ge", "9999-01-01t00:00:00.00z")) _begin_m = parse_date(msg.get("begin_le", "1900-01-01t00:00:00.00z")) placeholders = ','.join([str(x) for x in _kinds]) # "%s, %s, %s, ... %s" # log("placeholders {}".format(placeholders)) _cql = """SELECT device_uuid, uuid, source, kind, begin_time, end_time, properties FROM events_by_device WHERE device_uuid = {} AND hour = '{}' AND kind IN ({}) """.format(_uuid, hour, placeholders) # log(_cql) query = SimpleStatement(_cql, consistency_level=ConsistencyLevel.LOCAL_ONE, fetch_size=FETCH_SIZE) rows = session.execute(query) _events = [] try: for row in rows: if row.begin_time > _end_m and row.end_time < _begin_m: _ev = { "id": hash(row[0]), "device_uuid": str(row[0]), "camera_uid": str(row[0]), "uuid": str(row[1]), "source": row[2], "kind": row[3], "begin_time": format_date(row[4]), "end_time": format_date(row[5]), "properties": json.loads(row[6]) } _events.append(_ev) # log("row: {}".format(row)) # log("len of events {}".format(len(_events))) return _events except ReadTimeout: raise Exception("Read timeout")
def make_response_future(self, session): query = SimpleStatement("SELECT * FROM foo") message = QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE) return ResponseFuture(session, message, query, 1)
def get_long_events(args): session, _uuid, msg = args _source, _kinds = get_source_kinds(msg.get("source"), msg.get("kinds")) # log("get_source_kinds {} {}".format(_source, _kinds)) if _source is None or _kinds is None: return "bad request: source {}, kinds {}".format(_source, _kinds) _end_m = parse_date(msg.get("end_ge", "9999-01-01t00:00:00.00z")) # _begin_m = parse_date(msg.get("begin_le", "1900-01-01t00:00:00.00z")) placeholders = ','.join([str(x) for x in _kinds]) # "%s, %s, %s, ... %s" # log("{} {}".format(_end_m, _begin_m)) _cql = """SELECT device_uuid, uuid, source, kind, begin_time, end_time, properties FROM events_by_device_up10min WHERE device_uuid = {} AND kind IN ({}) AND begin_time < '{}' AND begin_time > '{}' """.format(_uuid, placeholders, format_hour(_end_m + timedelta(hours=1)), format_hour(_end_m - timedelta(hours=MAX_UNFINISHED))) # log(_cql) query = SimpleStatement(_cql, consistency_level=ConsistencyLevel.LOCAL_ONE, fetch_size=FETCH_SIZE) rows = session.execute(query) _events = [] try: for row in rows: # log("begin_time {} _end_m {} end_time {} _begin_m {}".format(row.begin_time, _end_m, row.end_time, _begin_m)) if row.begin_time > _end_m: continue if row.end_time < _end_m: continue _ev = { "id": hash(row[0]), "device_uuid": str(row[0]), "camera_uid": str(row[0]), "uuid": str(row[1]), "source": row[2], "kind": row[3], "begin_time": format_date(row[4]), "end_time": format_date(row[5]), "properties": json.loads(row[6]) } _events.append(_ev) # log("row: {}".format(row)) # log("len of long{}".format(len(_events))) return sorted(_events, key=lambda i: parse2_date(i['begin_time']), reverse=True) except ReadTimeout: raise Exception("Read timeout")
session.execute( """ CREATE KEYSPACE IF NOT EXISTS customer_data WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1 }; """ ) session.execute( """ CREATE TABLE IF NOT EXISTS customer_data.customers ( customer_id int PRIMARY KEY, first_name text, last_name text, email text ); """ ) json_data = {} with open('data.json') as json_file: json_data = json.load(json_file) batch = BatchStatement() for data in json_data: customer_id = data['id'] first_name = data['first_name'] last_name = data['last_name'] email = data['email'] batch.add(SimpleStatement("INSERT INTO customer_data.customers (customer_id, first_name, last_name, email) VALUES (%s, %s, %s, %s)"), (customer_id, first_name, last_name, email)) session.execute(batch)
def read_from_table(sess, table_name): sess.execute_async(SimpleStatement("SELECT * FROM %s LIMIT %d" % (table_name, random.choice(range(1, 200))))) return
def test_read_repair_chance(self): """ @jira_ticket CASSANDRA-12368 """ # session is only used to setup & do schema modification. Actual data queries are done directly on # each node, using an exclusive connection and CL.ONE session = self.patient_cql_connection(self.cluster.nodelist()[0]) initial_replica, non_replicas = self.do_initial_setup(session) # To ensure read repairs are triggered, set the table property to 100% logger.debug("Setting table read repair chance to 1") session.execute( """ALTER TABLE alter_rf_test.t1 WITH read_repair_chance = 1;""") # Execute a query at CL.ONE on one of the nodes which was *not* the initial replica. It should trigger a # read repair because read_repair_chance == 1, and propagate the data to all 3 nodes. # Note: result of the read repair contains only the selected column (a), not all columns, so we won't expect # 'b' to have been fully repaired afterwards. logger.debug( "Executing 'SELECT a...' on non-initial replica to trigger read repair " + non_replicas[0].name) read_repair_session = self.patient_exclusive_cql_connection( non_replicas[0]) read_repair_session.execute( SimpleStatement("SELECT a FROM alter_rf_test.t1 WHERE k=1", consistency_level=ConsistencyLevel.ONE)) # Query each replica individually to ensure that read repair was triggered. We should expect that only # the initial replica has data for both the 'a' and 'b' columns. If the cluster is on > 3.4, the read repair # should only have affected the selected column (CASSANDRA-10655), so the other two replicas should only have # that data. # Note: we need to temporarily set read_repair_chance to 0 while we perform this check. logger.debug( "Setting table read repair chance to 0 while we verify each replica's data" ) session.execute( """ALTER TABLE alter_rf_test.t1 WITH read_repair_chance = 0;""") # The read repair is run in the background, so we spin while checking that the repair has completed value_skipping_disabled = True if self.cluster.version( ) < '3.4' else False retry_till_success(self.check_data_on_each_replica, expect_fully_repaired=value_skipping_disabled, initial_replica=initial_replica, timeout=30, bypassed_exception=NotRepairedException) # Re-enable global read repair and perform another query on a non-replica. This time the query selects all # columns so we also expect the value for 'b' to be repaired. logger.debug("Setting table read repair chance to 1") session.execute( """ALTER TABLE alter_rf_test.t1 WITH read_repair_chance = 1;""") logger.debug( "Executing 'SELECT *...' on non-initial replica to trigger read repair " + non_replicas[0].name) read_repair_session = self.patient_exclusive_cql_connection( non_replicas[0]) read_repair_session.execute( SimpleStatement("SELECT * FROM alter_rf_test.t1 WHERE k=1", consistency_level=ConsistencyLevel.ONE)) # Query each replica again to ensure that second read repair was triggered. This time, we expect the # data to be fully repaired (both 'a' and 'b' columns) by virtue of the query being 'SELECT *...' # As before, we turn off read repair before doing this check. logger.debug( "Setting table read repair chance to 0 while we verify each replica's data" ) session.execute( """ALTER TABLE alter_rf_test.t1 WITH read_repair_chance = 0;""") retry_till_success(self.check_data_on_each_replica, expect_fully_repaired=True, initial_replica=initial_replica, timeout=30, bypassed_exception=NotRepairedException)
def truncate_tables(self, session): statement = SimpleStatement("TRUNCATE users", ConsistencyLevel.ALL) session.execute(statement) statement = SimpleStatement("TRUNCATE counters", ConsistencyLevel.ALL) session.execute(statement)
def quorum(query_string): return SimpleStatement(query_string=query_string, consistency_level=ConsistencyLevel.QUORUM)
def insert_user(self, session, userid, age, consistency, serial_consistency=None): text = "INSERT INTO users (userid, firstname, lastname, age) VALUES ({}, 'first{}', 'last{}', {}) {}"\ .format(userid, userid, userid, age, "IF NOT EXISTS" if serial_consistency else "") statement = SimpleStatement(text, consistency_level=consistency, serial_consistency_level=serial_consistency) session.execute(statement)
def fire_query(query): query = SimpleStatement(q) session.execute(query)
def update_user(self, session, userid, age, consistency, serial_consistency=None, prev_age=None): text = "UPDATE users SET age = {} WHERE userid = {}".format(age, userid) if serial_consistency and prev_age: text = text + " IF age = {}".format(prev_age) statement = SimpleStatement(text, consistency_level=consistency, serial_consistency_level=serial_consistency) session.execute(statement)
def test_duplicate_metrics_per_cluster(self): """ Test to validate that cluster metrics names can't overlap. @since 3.6.0 @jira_ticket PYTHON-561 @expected_result metric names should not be allowed to be same. @test_category metrics """ cluster2 = Cluster( metrics_enabled=True, protocol_version=PROTOCOL_VERSION, monitor_reporting_enabled=False, execution_profiles={ EXEC_PROFILE_DEFAULT: ExecutionProfile(retry_policy=FallthroughRetryPolicy()) }) cluster3 = Cluster( metrics_enabled=True, protocol_version=PROTOCOL_VERSION, monitor_reporting_enabled=False, execution_profiles={ EXEC_PROFILE_DEFAULT: ExecutionProfile(retry_policy=FallthroughRetryPolicy()) }) # Ensure duplicate metric names are not allowed cluster2.metrics.set_stats_name("appcluster") cluster2.metrics.set_stats_name("appcluster") with self.assertRaises(ValueError): cluster3.metrics.set_stats_name("appcluster") cluster3.metrics.set_stats_name("devops") session2 = cluster2.connect(self.ks_name, wait_for_all_pools=True) session3 = cluster3.connect(self.ks_name, wait_for_all_pools=True) # Basic validation that naming metrics doesn't impact their segration or accuracy for i in range(10): query = SimpleStatement("SELECT * FROM {0}.{0}".format( self.ks_name), consistency_level=ConsistencyLevel.ALL) session2.execute(query) for i in range(5): query = SimpleStatement("SELECT * FROM {0}.{0}".format( self.ks_name), consistency_level=ConsistencyLevel.ALL) session3.execute(query) self.assertEqual( cluster2.metrics.get_stats()['request_timer']['count'], 10) self.assertEqual( cluster3.metrics.get_stats()['request_timer']['count'], 5) # Check scales to ensure they are appropriately named self.assertTrue("appcluster" in scales._Stats.stats.keys()) self.assertTrue("devops" in scales._Stats.stats.keys()) cluster2.shutdown() cluster3.shutdown()
def delete_user(self, session, userid, consistency): statement = SimpleStatement("DELETE FROM users where userid = {}".format(userid), consistency_level=consistency) session.execute(statement)
def read_failure_query(): assert_invalid(session, SimpleStatement( "select * from test where id in (1,2,3,4,5)", consistency_level=CL.ALL), expected=ReadFailure)
def update_counter(self, session, id, consistency, serial_consistency=None): text = "UPDATE counters SET c = c + 1 WHERE id = {}".format(id) statement = SimpleStatement(text, consistency_level=consistency, serial_consistency_level=serial_consistency) session.execute(statement) return statement
def tombstone_failure_threshold_message_test(self): """ Ensure nodes return an error message in case of TombstoneOverwhelmingExceptions rather than dropping the request. A drop makes the coordinator waits for the specified read_request_timeout_in_ms. @jira_ticket CASSANDRA-7886 """ have_v5_protocol = self.cluster.version() >= LooseVersion('3.10') self.allow_log_errors = True self.cluster.set_configuration_options( values={ 'tombstone_failure_threshold': 500, 'read_request_timeout_in_ms': 30000, # 30 seconds 'range_request_timeout_in_ms': 40000 } ) self.cluster.populate(3).start() node1, node2, node3 = self.cluster.nodelist() proto_version = 5 if have_v5_protocol else None session = self.patient_cql_connection(node1, protocol_version=proto_version) self.create_ks(session, 'test', 3) session.execute( "CREATE TABLE test ( " "id int, mytext text, col1 int, col2 int, col3 int, " "PRIMARY KEY (id, mytext) )" ) # Add data with tombstones values = map(lambda i: str(i), range(1000)) for value in values: session.execute(SimpleStatement( "insert into test (id, mytext, col1) values (1, '{}', null) ".format( value ), consistency_level=CL.ALL )) failure_msg = ("Scanned over.* tombstones.* query aborted") @timed(25) def read_failure_query(): try: session.execute(SimpleStatement("select * from test where id in (1,2,3,4,5)", consistency_level=CL.ALL)) except ReadFailure as exc: if have_v5_protocol: # at least one replica should have responded with a tombstone error self.assertIsNotNone(exc.error_code_map) self.assertEqual(0x0001, exc.error_code_map.values()[0]) except Exception: raise else: self.fail('Expected ReadFailure') read_failure_query() # In almost all cases, we should find the failure message on node1 within a few seconds. # If it is not on node1, we grep all logs, as it *absolutely* should be somewhere. # If we still cannot find it then, we fail the test, as this is a problem. try: node1.watch_log_for(failure_msg, timeout=5) except TimeoutError: failure = (node1.grep_log(failure_msg) or node2.grep_log(failure_msg) or node3.grep_log(failure_msg)) self.assertTrue(failure, ("Cannot find tombstone failure threshold error in log " "after failed query")) mark1 = node1.mark_log() mark2 = node2.mark_log() mark3 = node3.mark_log() @timed(35) def range_request_failure_query(): try: session.execute(SimpleStatement("select * from test", consistency_level=CL.ALL)) except ReadFailure as exc: if have_v5_protocol: # at least one replica should have responded with a tombstone error self.assertIsNotNone(exc.error_code_map) self.assertEqual(0x0001, exc.error_code_map.values()[0]) except Exception: raise else: self.fail('Expected ReadFailure') range_request_failure_query() # In almost all cases, we should find the failure message on node1 within a few seconds. # If it is not on node1, we grep all logs, as it *absolutely* should be somewhere. # If we still cannot find it then, we fail the test, as this is a problem. try: node1.watch_log_for(failure_msg, from_mark=mark1, timeout=5) except TimeoutError: failure = (node1.grep_log(failure_msg, from_mark=mark1) or node2.grep_log(failure_msg, from_mark=mark2) or node3.grep_log(failure_msg, from_mark=mark3)) self.assertTrue(failure, ("Cannot find tombstone failure threshold error in log " "after range_request_timeout_query"))
def short_read_test(self): """ @jira_ticket CASSANDRA-9460 """ cluster = self.cluster # Disable hinted handoff and set batch commit log so this doesn't # interfer with the test cluster.set_configuration_options(values={'hinted_handoff_enabled': False}) cluster.set_batch_commitlog(enabled=True) cluster.populate(3).start(wait_other_notice=True) node1, node2, node3 = cluster.nodelist() session = self.patient_cql_connection(node1) self.create_ks(session, 'ks', 3) self.create_cf(session, 'cf', read_repair=0.0) normal_key = 'normal' reversed_key = 'reversed' # Repeat this test 10 times to make it more easy to spot a null pointer exception caused by a race, see CASSANDRA-9460 for k in xrange(10): # insert 9 columns in two rows insert_columns(self, session, normal_key, 9) insert_columns(self, session, reversed_key, 9) # Delete 3 first columns (and 3 last columns, for the reversed version) with a different node dead each time for node, column_number_to_delete in zip(range(1, 4), range(3)): self.stop_node(node) self.delete(node, normal_key, column_number_to_delete) self.delete(node, reversed_key, 8 - column_number_to_delete) self.restart_node(node) # Query 3 firsts columns in normal order session = self.patient_cql_connection(node1, 'ks') query = SimpleStatement( 'SELECT c, v FROM cf WHERE key=\'k{}\' LIMIT 3'.format(normal_key), consistency_level=ConsistencyLevel.QUORUM) rows = list(session.execute(query)) res = rows assert_length_equal(res, 3) # value 0, 1 and 2 have been deleted for i in xrange(1, 4): self.assertEqual('value{}'.format(i + 2), res[i - 1][1]) # Query 3 firsts columns in reverse order session = self.patient_cql_connection(node1, 'ks') query = SimpleStatement( 'SELECT c, v FROM cf WHERE key=\'k{}\' ORDER BY c DESC LIMIT 3'.format(reversed_key), consistency_level=ConsistencyLevel.QUORUM) rows = list(session.execute(query)) res = rows assert_length_equal(res, 3) # value 6, 7 and 8 have been deleted for i in xrange(0, 3): self.assertEqual('value{}'.format(5 - i), res[i][1]) session.execute('TRUNCATE cf')
def alter_rf_and_run_read_repair_test(self): """ @jira_ticket CASSANDRA-10655 @jira_ticket CASSANDRA-10657 Test that querying only a subset of all the columns in a row doesn't confuse read-repair to avoid the problem described in CASSANDRA-10655. """ session = self.patient_cql_connection(self.cluster.nodelist()[0]) session.execute("""CREATE KEYSPACE alter_rf_test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};""" ) session.execute( "CREATE TABLE alter_rf_test.t1 (k int PRIMARY KEY, a int, b int);") session.execute( "INSERT INTO alter_rf_test.t1 (k, a, b) VALUES (1, 1, 1);") cl_one_stmt = SimpleStatement( "SELECT * FROM alter_rf_test.t1 WHERE k=1", consistency_level=ConsistencyLevel.ONE) # identify the initial replica and trigger a flush to ensure reads come from sstables initial_replica, non_replicas = self.identify_initial_placement( 'alter_rf_test', 't1', 1) debug("At RF=1 replica for data is " + initial_replica.name) initial_replica.flush() # At RF=1, it shouldn't matter which node we query, as the actual data should always come from the # initial replica when reading at CL ONE for n in self.cluster.nodelist(): debug("Checking " + n.name) session = self.patient_exclusive_cql_connection(n) assert_one(session, "SELECT * FROM alter_rf_test.t1 WHERE k=1", [1, 1, 1], cl=ConsistencyLevel.ONE) # Alter so RF=n but don't repair, then execute a query which selects only a subset of the columns. Run this at # CL ALL on one of the nodes which doesn't currently have the data, triggering a read repair. # The expectation will be that every replicas will have been repaired for that column (but we make no assumptions # on the other columns). debug("Changing RF from 1 to 3") session.execute("""ALTER KEYSPACE alter_rf_test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};""" ) cl_all_stmt = SimpleStatement( "SELECT a FROM alter_rf_test.t1 WHERE k=1", consistency_level=ConsistencyLevel.ALL) debug( "Executing SELECT on non-initial replica to trigger read repair " + non_replicas[0].name) read_repair_session = self.patient_exclusive_cql_connection( non_replicas[0]) # result of the CL ALL query contains only the selected column assert_one(read_repair_session, "SELECT a FROM alter_rf_test.t1 WHERE k=1", [1], cl=ConsistencyLevel.ALL) # Check the results of the read repair by querying each replica again at CL ONE debug("Re-running SELECTs at CL ONE to verify read repair") for n in self.cluster.nodelist(): debug("Checking " + n.name) session = self.patient_exclusive_cql_connection(n) res = rows_to_list(session.execute(cl_one_stmt)) # Column a must be 1 everywhere, and column b must be either 1 or None everywhere self.assertIn(res[0][:2], [[1, 1], [1, None]]) # Now query at ALL but selecting all columns query = "SELECT * FROM alter_rf_test.t1 WHERE k=1" debug( "Executing SELECT on non-initial replica to trigger read repair " + non_replicas[0].name) read_repair_session = self.patient_exclusive_cql_connection( non_replicas[0]) assert_one(session, query, [1, 1, 1], cl=ConsistencyLevel.ALL) # Check all replica is fully up to date debug("Re-running SELECTs at CL ONE to verify read repair") for n in self.cluster.nodelist(): debug("Checking " + n.name) session = self.patient_exclusive_cql_connection(n) assert_one(session, query, [1, 1, 1], cl=ConsistencyLevel.ONE)
def execute(query, params=None, consistency_level=None): params = params or {} consistency_level = get_consistency_level(consistency_level) session = get_connection_pool() query = SimpleStatement(query, consistency_level=consistency_level) return session.execute(query, parameters=params)
def upload_file(self): """ This will upload all the selected csv/xml files in the database for the given algorithm. :return: Json Response """ try: assert self._db_connection, { STATUS_KEY: HTTP_500_INTERNAL_SERVER_ERROR, MESSAGE_KEY: DB_ERROR } batch = BatchStatement() error = False error_message = None FLAG = 0 algo_tag = "TMT" """if not self.files: error = True error_message = "No files to upload" file_names_list = self.file_names # select_query = SELECT_ALGORITHM_NAME_QUERY.format(self.algo_name, ",".join(map(lambda x: "'" + x + "'", file_names_list))) select_query = "SELECT algorithm_name from csv.configuration_details WHERE algorithm_name = '{}' ALLOW FILTERING".format( self.algo_name) result_set = self._csql_session.execute(select_query) if result_set[0]['count'] == 0 or result_set[0]['count'] < len(file_names_list): error_message = "Please give the existing algorithm or file name" return JsonResponse({MESSAGE_KEY: error_message}, status=HTTP_500_INTERNAL_SERVER_ERROR)""" for file in self.files: """if file.name not in self.file_names: error = True error_message = "Uploaded file name("+file.name+") not found in given file name list" break""" description = None if file.name in self.description: description = self.description[file.name] # print(self.description[f.name]) LAST_MODIFIED_DATE = str(round(time.time() * 1000)) #file_data = str(file.read(), 'utf-8') extension = os.path.splitext(file.name)[1] json_data = "" if extension == ".csv": file_data = pandas.read_csv(file, encoding='ISO-8859-1') json_data = file_data.to_json() elif extension == ".xml": file_data = et.parse(file) xml_str = ElementTree.tostring(file_data.getroot(), encoding='unicode') json_data = json.dumps(xmltodict.parse(xml_str)) elif extension == ".joblib": #json_data = joblib.load((os.path.realpath(file.name()))) #print(json_data) #print(file.) json_datas = joblib.load(file) # json_datas = pickle.load(open(file, "rb")) print(json_datas) json_data = escape(str(json_datas)) print(json_data) # json_data = json.dumps(file) # json_datas = joblib.load(file) # json_datas = pickle.load(open(file, "rb")) # print(json_datas) # json_data = escape(str(json_datas)) # print(json_data) # json_data = file #print(json_data) #print("ggggggg") #json_data = file.read().decode() # insert query into cassandra table # insert_query = """ INSERT into csv.configuration_details (algorithm_name, file_param_name, description, """ \ # """file_content, last_modified_date, type, flag) values ('{}', '{}', '{}', {}, {}, 2, {})""".format( # self.algo_name, # file.name, # description, # "textAsBlob('" + json_data + "')", # LAST_MODIFIED_DATE, # FLAG) insert_query = """ INSERT into {}.{} (algorithm_name, file_param_name, description, """ \ """file_content, last_modified_date, type, flag, algo_tag) values ('{}', '{}', '{}', {}, {}, 2, {}, '{}')""".format(NAME, TABLE_NAME, self.algo_name, file.name, description, "textAsBlob('" + json_data + "')", LAST_MODIFIED_DATE, FLAG, algo_tag) # batch insert initialisation batch.add(SimpleStatement(insert_query)) if error is True: return JsonResponse({MESSAGE_KEY: error_message}, status=HTTP_500_INTERNAL_SERVER_ERROR) self._csql_session.execute(batch, timeout=200.0) return JsonResponse({MESSAGE_KEY: UPLOADED_SUCCESSFULLY}, safe=False) except AssertionError as e: print(e) log_error(e) return JsonResponse({MESSAGE_KEY: e.args[0][MESSAGE_KEY]}, status=e.args[0][STATUS_KEY]) except Exception as e: print("Error while inserting data - " + str(e)) log_error(traceback.format_exc()) return JsonResponse( {MESSAGE_KEY: EXCEPTION_CAUSE.format(traceback.format_exc())}, status=HTTP_500_INTERNAL_SERVER_ERROR)
def remote_query_test(self): """ Check that a query running on a node other than the coordinator times out: - populate the cluster with 2 nodes - set a 1-second read timeout - start one node without having it join the ring - start the other node with read_iteration_delay set to 1.5 seconds - (this will cause read queries to take longer than the read timeout) - CREATE a table - INSERT 5000 rows on a session on the node that is not a member of the ring - run SELECT statements and assert they fail # TODO refactor SELECT statements: # - run the statements in a loop to reduce duplication # - watch the log after each query # - assert we raise the right error """ cluster = self.cluster cluster.set_configuration_options( values={'read_request_timeout_in_ms': 1000}) cluster.populate(2) node1, node2 = cluster.nodelist() node1.start(wait_for_binary_proto=True, join_ring=False) # ensure other node executes queries node2.start(wait_for_binary_proto=True, jvm_args=[ "-Dcassandra.monitoring_check_interval_ms=50", "-Dcassandra.test.read_iteration_delay_ms=1500" ]) # see above for explanation session = self.patient_exclusive_cql_connection(node1) self.create_ks(session, 'ks', 1) session.execute(""" CREATE TABLE test2 ( id int, col int, val text, PRIMARY KEY(id, col) ); """) for i, j in itertools.product(range(500), range(10)): session.execute( "INSERT INTO test2 (id, col, val) VALUES ({}, {}, 'foo')". format(i, j)) mark = node2.mark_log() statement = SimpleStatement("SELECT * from test2", consistency_level=ConsistencyLevel.ONE, retry_policy=FallthroughRetryPolicy()) assert_unavailable(lambda c: debug(c.execute(statement)), session) statement = SimpleStatement("SELECT * from test2 where id = 1", consistency_level=ConsistencyLevel.ONE, retry_policy=FallthroughRetryPolicy()) assert_unavailable(lambda c: debug(c.execute(statement)), session) statement = SimpleStatement( "SELECT * from test2 where id IN (1, 10, 20) AND col < 10", consistency_level=ConsistencyLevel.ONE, retry_policy=FallthroughRetryPolicy()) assert_unavailable(lambda c: debug(c.execute(statement)), session) statement = SimpleStatement( "SELECT * from test2 where col > 5 ALLOW FILTERING", consistency_level=ConsistencyLevel.ONE, retry_policy=FallthroughRetryPolicy()) assert_unavailable(lambda c: debug(c.execute(statement)), session) node2.watch_log_for("operations timed out", from_mark=mark, timeout=60)