print("\nperformed %d actions in %s mode\n write actions: %d\n read actions: %d" % (actions, args.mode, write_actions, read_actions))
sys.exit(0)
signal.signal(signal.SIGINT, sigint_handler)
session = setup_session(args.cluster_ip, args.keyspace)
if not args.keyspace:
args.keyspace = select_random_keyspace(session)
if random_table:
args.table = select_random_table(session, args.keyspace)
session.set_keyspace(args.keyspace)
print("Selected table %s from keyspace %s" % (args.table, args.keyspace))
print("stressing database by sending {} queries every {} seconds...".format(args.mode, args.pause))
fetchStatement = SimpleStatement("SELECT * FROM " + args.table)
rows = session.execute(fetchStatement)
column_names = session.cluster.metadata.keyspaces[args.keyspace].tables[args.table].columns.keys()
if args.mode == "READ":
while True:
read_from_table(session, args.table)
actions += 1
pause(args.pause)
if args.mode == "INSERT":
rows = session.execute(fetchStatement)
while True:
insert_random_rows(session, args.table, rows.current_rows, column_names, args.batch_size)
actions += 1
write_actions += 1
pause(args.pause)
def counter_consistency_test(self):
"""
Do a bunch of writes with ONE, read back with ALL and check results.
"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
session = self.patient_cql_connection(node1)
create_ks(session, 'counter_tests', 3)
stmt = """
CREATE TABLE counter_table (
id uuid PRIMARY KEY,
counter_one COUNTER,
counter_two COUNTER,
)
"""
session.execute(stmt)
counters = []
# establish 50 counters (2x25 rows)
for i in xrange(25):
_id = str(uuid.uuid4())
counters.append({_id: {'counter_one': 1, 'counter_two': 1}})
query = SimpleStatement("""
UPDATE counter_table
SET counter_one = counter_one + 1, counter_two = counter_two + 1
where id = {uuid}""".format(uuid=_id),
consistency_level=ConsistencyLevel.ONE)
session.execute(query)
# increment a bunch of counters with CL.ONE
for i in xrange(10000):
counter = counters[random.randint(0, len(counters) - 1)]
counter_id = counter.keys()[0]
query = SimpleStatement("""
UPDATE counter_table
SET counter_one = counter_one + 2
where id = {uuid}""".format(uuid=counter_id),
consistency_level=ConsistencyLevel.ONE)
session.execute(query)
query = SimpleStatement("""
UPDATE counter_table
SET counter_two = counter_two + 10
where id = {uuid}""".format(uuid=counter_id),
consistency_level=ConsistencyLevel.ONE)
session.execute(query)
query = SimpleStatement("""
UPDATE counter_table
SET counter_one = counter_one - 1
where id = {uuid}""".format(uuid=counter_id),
consistency_level=ConsistencyLevel.ONE)
session.execute(query)
query = SimpleStatement("""
UPDATE counter_table
SET counter_two = counter_two - 5
where id = {uuid}""".format(uuid=counter_id),
consistency_level=ConsistencyLevel.ONE)
session.execute(query)
# update expectations to match (assumed) db state
counter[counter_id]['counter_one'] += 1
counter[counter_id]['counter_two'] += 5
# let's verify the counts are correct, using CL.ALL
for counter_dict in counters:
counter_id = counter_dict.keys()[0]
query = SimpleStatement("""
SELECT counter_one, counter_two
FROM counter_table WHERE id = {uuid}
""".format(uuid=counter_id),
consistency_level=ConsistencyLevel.ALL)
rows = list(session.execute(query))
counter_one_actual, counter_two_actual = rows[0]
self.assertEqual(counter_one_actual,
counter_dict[counter_id]['counter_one'])
self.assertEqual(counter_two_actual,
counter_dict[counter_id]['counter_two'])
cluster = Cluster(
seeds,
load_balancing_policy=WhiteListRoundRobinPolicy(seeds),
control_connection_timeout=180,
connect_timeout=180
)
session = cluster.connect()
session.row_factory = dict_factory
session.default_fetch_size = 100
session.default_timeout = 180
count = 0
for key in keys:
#print key[0], key[1]
stat = "select version from aosmd_us_central_1.object_versions where bucket = '%s' and key = '%s';" % (key[0], key[1])
query = SimpleStatement(stat, consistency_level=ConsistencyLevel.LOCAL_QUORUM)
res = session.execute(query)
count = 0
for r in res:
count += 1
try:
version = r['version']
stat1 = "select * from aosmd_us_central_1.object_versions where bucket = '%s' and key = '%s' and version = '%s';" % (key[0], key[1], version)
print stat1
query1 = SimpleStatement(stat, consistency_level=ConsistencyLevel.LOCAL_QUORUM)
ret = session.execute(query1)
except Exception, ex:
print '%s failed' % version
else:
print '%s success' % version
print count
def tombstone_failure_threshold_message_test(self):
"""
Ensure nodes return an error message in case of TombstoneOverwhelmingExceptions rather
than dropping the request. A drop makes the coordinator waits for the specified
read_request_timeout_in_ms.
@jira_ticket CASSANDRA-7886
"""
self.allow_log_errors = True
self.cluster.set_configuration_options(
values={
'tombstone_failure_threshold': 500,
'read_request_timeout_in_ms': 30000, # 30 seconds
'range_request_timeout_in_ms': 40000
})
self.cluster.populate(3).start()
node1, node2, node3 = self.cluster.nodelist()
session = self.patient_cql_connection(node1)
self.create_ks(session, 'test', 3)
session.execute("CREATE TABLE test ( "
"id int, mytext text, col1 int, col2 int, col3 int, "
"PRIMARY KEY (id, mytext) )")
# Add data with tombstones
values = map(lambda i: str(i), range(1000))
for value in values:
session.execute(
SimpleStatement(
"insert into test (id, mytext, col1) values (1, '{}', null) "
.format(value),
consistency_level=CL.ALL))
failure_msg = ("Scanned over.* tombstones.* query aborted")
@timed(25)
def read_failure_query():
assert_invalid(session,
SimpleStatement(
"select * from test where id in (1,2,3,4,5)",
consistency_level=CL.ALL),
expected=ReadFailure)
read_failure_query()
failure = (node1.grep_log(failure_msg) or node2.grep_log(failure_msg)
or node3.grep_log(failure_msg))
self.assertTrue(
failure, ("Cannot find tombstone failure threshold error in log "
"after failed query"))
mark1 = node1.mark_log()
mark2 = node2.mark_log()
mark3 = node3.mark_log()
@timed(35)
def range_request_failure_query():
assert_invalid(session,
SimpleStatement("select * from test",
consistency_level=CL.ALL),
expected=ReadFailure)
range_request_failure_query()
failure = (
node1.watch_log_for(failure_msg, from_mark=mark1, timeout=5)
or node2.watch_log_for(failure_msg, from_mark=mark2, timeout=5)
or node3.watch_log_for(failure_msg, from_mark=mark3, timeout=5))
self.assertTrue(
failure, ("Cannot find tombstone failure threshold error in log "
"after range_request_timeout_query"))
def range_request_failure_query():
assert_invalid(session,
SimpleStatement("select * from test",
consistency_level=CL.ALL),
expected=ReadFailure)
from cassandra import ConsistencyLevel
from cassandra.cluster import Cluster
from cassandra.query import SimpleStatement
transaction2 = SimpleStatement(
"BEGIN BATCH INSERT INTO vacancybot.users(first_name, last_name, balance, living_wage, birthday, city, skills) VALUES('Ioan', 'Petrov', 1800, 5000,toTimeStamp(toDate('2019-09-29 00:00:00+0000')), 'Kiev', [{name:'potions'}]) UPDATE vacancybot.vacancy SET salary = 24000 WHERE profession = 'Dancing teacher' AND name = 'Dancing teacher' and created_at = toTimeStamp(toDate('2019-09-29 00:00:00+0000')) APPLY BATCH",
consistency_level=ConsistencyLevel.ANY)
cluster = Cluster()
session = cluster.connect('vacancybot')
session.execute(transaction2)
def test_metrics_per_cluster(self):
"""
Test to validate that metrics can be scopped to invdividual clusters
@since 3.6.0
@jira_ticket PYTHON-561
@expected_result metrics should be scopped to a cluster level
@test_category metrics
"""
cluster2 = Cluster(
metrics_enabled=True,
protocol_version=PROTOCOL_VERSION,
execution_profiles={
EXEC_PROFILE_DEFAULT:
ExecutionProfile(retry_policy=FallthroughRetryPolicy())
})
cluster2.connect(self.ks_name, wait_for_all_pools=True)
self.assertEqual(len(cluster2.metadata.all_hosts()), 3)
query = SimpleStatement("SELECT * FROM {0}.{0}".format(self.ks_name),
consistency_level=ConsistencyLevel.ALL)
self.session.execute(query)
# Pause node so it shows as unreachable to coordinator
get_node(1).pause()
try:
# Test write
query = SimpleStatement(
"INSERT INTO {0}.{0} (k, v) VALUES (2, 2)".format(
self.ks_name),
consistency_level=ConsistencyLevel.ALL)
with self.assertRaises(WriteTimeout):
self.session.execute(query, timeout=None)
finally:
get_node(1).resume()
# Change the scales stats_name of the cluster2
cluster2.metrics.set_stats_name('cluster2-metrics')
stats_cluster1 = self.cluster.metrics.get_stats()
stats_cluster2 = cluster2.metrics.get_stats()
# Test direct access to stats
self.assertEqual(1, self.cluster.metrics.stats.write_timeouts)
self.assertEqual(0, cluster2.metrics.stats.write_timeouts)
# Test direct access to a child stats
self.assertNotEqual(0.0, self.cluster.metrics.request_timer['mean'])
self.assertEqual(0.0, cluster2.metrics.request_timer['mean'])
# Test access via metrics.get_stats()
self.assertNotEqual(0.0, stats_cluster1['request_timer']['mean'])
self.assertEqual(0.0, stats_cluster2['request_timer']['mean'])
# Test access by stats_name
self.assertEqual(
0.0,
scales.getStats()['cluster2-metrics']['request_timer']['mean'])
cluster2.shutdown()
def execute_async( self, query, params = {} ):
if type( query ) is str or type( query ) is unicode:
query = SimpleStatement( query, consistency_level = self.consistency )
return self.cur.execute_async( query, params )
# Parsing args
hostname = sys.argv[1]
#username = sys.argv[2]
#password = sys.argv[3]
desired_response_counter = int(sys.argv[2])
# Logging in
#auth_provider = PlainTextAuthProvider(username=username, password=password)
cluster = Cluster(
[hostname],
# auth_provider=auth_provider,
load_balancing_policy=DCAwareRoundRobinPolicy(local_dc='us-east'))
session = cluster.connect('replicated')
# Predefining queries
truncate_test = SimpleStatement("TRUNCATE replicated.test",
consistency_level=ConsistencyLevel.ALL)
truncate_test_count = SimpleStatement("TRUNCATE replicated.test_count",
consistency_level=ConsistencyLevel.ALL)
insert_into_test = SimpleStatement(
"""
INSERT INTO replicated.test (id, insertion_date, desired_response_counter, some_data)
VALUES (%s, now(), %s, 'dummy');
""",
consistency_level=ConsistencyLevel.LOCAL_ONE)
while True:
# Inserting new test value
next_uuid = uuid.uuid1()
response_counter = 0
session.execute(insert_into_test, (next_uuid, desired_response_counter))
# Waiting for all responses to arrive
def get_event_by_device0(args):
session, _uuid, hour, msg = args
_source, _kinds = get_source_kinds(msg.get("source"), msg.get("kinds"))
# log("get_source_kinds {} {}".format(_source, _kinds))
if _source is None or _kinds is None:
return "bad request: source {}, kinds {}".format(_source, _kinds)
_end_m = parse_date(msg.get("end_ge", "9999-01-01t00:00:00.00z"))
_begin_m = parse_date(msg.get("begin_le", "1900-01-01t00:00:00.00z"))
placeholders = ','.join([str(x) for x in _kinds]) # "%s, %s, %s, ... %s"
# log("placeholders {}".format(placeholders))
_cql = """SELECT device_uuid,
uuid,
source,
kind,
begin_time,
end_time,
properties
FROM events_by_device
WHERE device_uuid = {}
AND hour = '{}'
AND kind IN ({})
""".format(_uuid, hour, placeholders)
# log(_cql)
query = SimpleStatement(_cql,
consistency_level=ConsistencyLevel.LOCAL_ONE,
fetch_size=FETCH_SIZE)
rows = session.execute(query)
_events = []
try:
for row in rows:
if row.begin_time > _end_m and row.end_time < _begin_m:
_ev = {
"id": hash(row[0]),
"device_uuid": str(row[0]),
"camera_uid": str(row[0]),
"uuid": str(row[1]),
"source": row[2],
"kind": row[3],
"begin_time": format_date(row[4]),
"end_time": format_date(row[5]),
"properties": json.loads(row[6])
}
_events.append(_ev)
# log("row: {}".format(row))
# log("len of events {}".format(len(_events)))
return _events
except ReadTimeout:
raise Exception("Read timeout")
def make_response_future(self, session):
query = SimpleStatement("SELECT * FROM foo")
message = QueryMessage(query=query,
consistency_level=ConsistencyLevel.ONE)
return ResponseFuture(session, message, query, 1)
def get_long_events(args):
session, _uuid, msg = args
_source, _kinds = get_source_kinds(msg.get("source"), msg.get("kinds"))
# log("get_source_kinds {} {}".format(_source, _kinds))
if _source is None or _kinds is None:
return "bad request: source {}, kinds {}".format(_source, _kinds)
_end_m = parse_date(msg.get("end_ge", "9999-01-01t00:00:00.00z"))
# _begin_m = parse_date(msg.get("begin_le", "1900-01-01t00:00:00.00z"))
placeholders = ','.join([str(x) for x in _kinds]) # "%s, %s, %s, ... %s"
# log("{} {}".format(_end_m, _begin_m))
_cql = """SELECT device_uuid,
uuid,
source,
kind,
begin_time,
end_time,
properties
FROM events_by_device_up10min
WHERE device_uuid = {}
AND kind IN ({})
AND begin_time < '{}'
AND begin_time > '{}'
""".format(_uuid, placeholders,
format_hour(_end_m + timedelta(hours=1)),
format_hour(_end_m - timedelta(hours=MAX_UNFINISHED)))
# log(_cql)
query = SimpleStatement(_cql,
consistency_level=ConsistencyLevel.LOCAL_ONE,
fetch_size=FETCH_SIZE)
rows = session.execute(query)
_events = []
try:
for row in rows:
# log("begin_time {} _end_m {} end_time {} _begin_m {}".format(row.begin_time, _end_m, row.end_time, _begin_m))
if row.begin_time > _end_m:
continue
if row.end_time < _end_m:
continue
_ev = {
"id": hash(row[0]),
"device_uuid": str(row[0]),
"camera_uid": str(row[0]),
"uuid": str(row[1]),
"source": row[2],
"kind": row[3],
"begin_time": format_date(row[4]),
"end_time": format_date(row[5]),
"properties": json.loads(row[6])
}
_events.append(_ev)
# log("row: {}".format(row))
# log("len of long{}".format(len(_events)))
return sorted(_events,
key=lambda i: parse2_date(i['begin_time']),
reverse=True)
except ReadTimeout:
raise Exception("Read timeout")
session.execute(
"""
CREATE KEYSPACE IF NOT EXISTS customer_data WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1 };
"""
)
session.execute(
"""
CREATE TABLE IF NOT EXISTS customer_data.customers ( customer_id int PRIMARY KEY, first_name text, last_name text, email text );
"""
)
json_data = {}
with open('data.json') as json_file:
json_data = json.load(json_file)
batch = BatchStatement()
for data in json_data:
customer_id = data['id']
first_name = data['first_name']
last_name = data['last_name']
email = data['email']
batch.add(SimpleStatement("INSERT INTO customer_data.customers (customer_id, first_name, last_name, email) VALUES (%s, %s, %s, %s)"), (customer_id, first_name, last_name, email))
session.execute(batch)
def read_from_table(sess, table_name):
sess.execute_async(SimpleStatement("SELECT * FROM %s LIMIT %d" % (table_name, random.choice(range(1, 200)))))
return
def test_read_repair_chance(self):
"""
@jira_ticket CASSANDRA-12368
"""
# session is only used to setup & do schema modification. Actual data queries are done directly on
# each node, using an exclusive connection and CL.ONE
session = self.patient_cql_connection(self.cluster.nodelist()[0])
initial_replica, non_replicas = self.do_initial_setup(session)
# To ensure read repairs are triggered, set the table property to 100%
logger.debug("Setting table read repair chance to 1")
session.execute(
"""ALTER TABLE alter_rf_test.t1 WITH read_repair_chance = 1;""")
# Execute a query at CL.ONE on one of the nodes which was *not* the initial replica. It should trigger a
# read repair because read_repair_chance == 1, and propagate the data to all 3 nodes.
# Note: result of the read repair contains only the selected column (a), not all columns, so we won't expect
# 'b' to have been fully repaired afterwards.
logger.debug(
"Executing 'SELECT a...' on non-initial replica to trigger read repair "
+ non_replicas[0].name)
read_repair_session = self.patient_exclusive_cql_connection(
non_replicas[0])
read_repair_session.execute(
SimpleStatement("SELECT a FROM alter_rf_test.t1 WHERE k=1",
consistency_level=ConsistencyLevel.ONE))
# Query each replica individually to ensure that read repair was triggered. We should expect that only
# the initial replica has data for both the 'a' and 'b' columns. If the cluster is on > 3.4, the read repair
# should only have affected the selected column (CASSANDRA-10655), so the other two replicas should only have
# that data.
# Note: we need to temporarily set read_repair_chance to 0 while we perform this check.
logger.debug(
"Setting table read repair chance to 0 while we verify each replica's data"
)
session.execute(
"""ALTER TABLE alter_rf_test.t1 WITH read_repair_chance = 0;""")
# The read repair is run in the background, so we spin while checking that the repair has completed
value_skipping_disabled = True if self.cluster.version(
) < '3.4' else False
retry_till_success(self.check_data_on_each_replica,
expect_fully_repaired=value_skipping_disabled,
initial_replica=initial_replica,
timeout=30,
bypassed_exception=NotRepairedException)
# Re-enable global read repair and perform another query on a non-replica. This time the query selects all
# columns so we also expect the value for 'b' to be repaired.
logger.debug("Setting table read repair chance to 1")
session.execute(
"""ALTER TABLE alter_rf_test.t1 WITH read_repair_chance = 1;""")
logger.debug(
"Executing 'SELECT *...' on non-initial replica to trigger read repair "
+ non_replicas[0].name)
read_repair_session = self.patient_exclusive_cql_connection(
non_replicas[0])
read_repair_session.execute(
SimpleStatement("SELECT * FROM alter_rf_test.t1 WHERE k=1",
consistency_level=ConsistencyLevel.ONE))
# Query each replica again to ensure that second read repair was triggered. This time, we expect the
# data to be fully repaired (both 'a' and 'b' columns) by virtue of the query being 'SELECT *...'
# As before, we turn off read repair before doing this check.
logger.debug(
"Setting table read repair chance to 0 while we verify each replica's data"
)
session.execute(
"""ALTER TABLE alter_rf_test.t1 WITH read_repair_chance = 0;""")
retry_till_success(self.check_data_on_each_replica,
expect_fully_repaired=True,
initial_replica=initial_replica,
timeout=30,
bypassed_exception=NotRepairedException)
def truncate_tables(self, session):
statement = SimpleStatement("TRUNCATE users", ConsistencyLevel.ALL)
session.execute(statement)
statement = SimpleStatement("TRUNCATE counters", ConsistencyLevel.ALL)
session.execute(statement)
def quorum(query_string):
return SimpleStatement(query_string=query_string,
consistency_level=ConsistencyLevel.QUORUM)
def insert_user(self, session, userid, age, consistency, serial_consistency=None):
text = "INSERT INTO users (userid, firstname, lastname, age) VALUES ({}, 'first{}', 'last{}', {}) {}"\
.format(userid, userid, userid, age, "IF NOT EXISTS" if serial_consistency else "")
statement = SimpleStatement(text, consistency_level=consistency, serial_consistency_level=serial_consistency)
session.execute(statement)
def fire_query(query):
query = SimpleStatement(q)
session.execute(query)
def update_user(self, session, userid, age, consistency, serial_consistency=None, prev_age=None):
text = "UPDATE users SET age = {} WHERE userid = {}".format(age, userid)
if serial_consistency and prev_age:
text = text + " IF age = {}".format(prev_age)
statement = SimpleStatement(text, consistency_level=consistency, serial_consistency_level=serial_consistency)
session.execute(statement)
def test_duplicate_metrics_per_cluster(self):
"""
Test to validate that cluster metrics names can't overlap.
@since 3.6.0
@jira_ticket PYTHON-561
@expected_result metric names should not be allowed to be same.
@test_category metrics
"""
cluster2 = Cluster(
metrics_enabled=True,
protocol_version=PROTOCOL_VERSION,
monitor_reporting_enabled=False,
execution_profiles={
EXEC_PROFILE_DEFAULT:
ExecutionProfile(retry_policy=FallthroughRetryPolicy())
})
cluster3 = Cluster(
metrics_enabled=True,
protocol_version=PROTOCOL_VERSION,
monitor_reporting_enabled=False,
execution_profiles={
EXEC_PROFILE_DEFAULT:
ExecutionProfile(retry_policy=FallthroughRetryPolicy())
})
# Ensure duplicate metric names are not allowed
cluster2.metrics.set_stats_name("appcluster")
cluster2.metrics.set_stats_name("appcluster")
with self.assertRaises(ValueError):
cluster3.metrics.set_stats_name("appcluster")
cluster3.metrics.set_stats_name("devops")
session2 = cluster2.connect(self.ks_name, wait_for_all_pools=True)
session3 = cluster3.connect(self.ks_name, wait_for_all_pools=True)
# Basic validation that naming metrics doesn't impact their segration or accuracy
for i in range(10):
query = SimpleStatement("SELECT * FROM {0}.{0}".format(
self.ks_name),
consistency_level=ConsistencyLevel.ALL)
session2.execute(query)
for i in range(5):
query = SimpleStatement("SELECT * FROM {0}.{0}".format(
self.ks_name),
consistency_level=ConsistencyLevel.ALL)
session3.execute(query)
self.assertEqual(
cluster2.metrics.get_stats()['request_timer']['count'], 10)
self.assertEqual(
cluster3.metrics.get_stats()['request_timer']['count'], 5)
# Check scales to ensure they are appropriately named
self.assertTrue("appcluster" in scales._Stats.stats.keys())
self.assertTrue("devops" in scales._Stats.stats.keys())
cluster2.shutdown()
cluster3.shutdown()
def delete_user(self, session, userid, consistency):
statement = SimpleStatement("DELETE FROM users where userid = {}".format(userid), consistency_level=consistency)
session.execute(statement)
def read_failure_query():
assert_invalid(session,
SimpleStatement(
"select * from test where id in (1,2,3,4,5)",
consistency_level=CL.ALL),
expected=ReadFailure)
def update_counter(self, session, id, consistency, serial_consistency=None):
text = "UPDATE counters SET c = c + 1 WHERE id = {}".format(id)
statement = SimpleStatement(text, consistency_level=consistency, serial_consistency_level=serial_consistency)
session.execute(statement)
return statement
def tombstone_failure_threshold_message_test(self):
"""
Ensure nodes return an error message in case of TombstoneOverwhelmingExceptions rather
than dropping the request. A drop makes the coordinator waits for the specified
read_request_timeout_in_ms.
@jira_ticket CASSANDRA-7886
"""
have_v5_protocol = self.cluster.version() >= LooseVersion('3.10')
self.allow_log_errors = True
self.cluster.set_configuration_options(
values={
'tombstone_failure_threshold': 500,
'read_request_timeout_in_ms': 30000, # 30 seconds
'range_request_timeout_in_ms': 40000
}
)
self.cluster.populate(3).start()
node1, node2, node3 = self.cluster.nodelist()
proto_version = 5 if have_v5_protocol else None
session = self.patient_cql_connection(node1, protocol_version=proto_version)
self.create_ks(session, 'test', 3)
session.execute(
"CREATE TABLE test ( "
"id int, mytext text, col1 int, col2 int, col3 int, "
"PRIMARY KEY (id, mytext) )"
)
# Add data with tombstones
values = map(lambda i: str(i), range(1000))
for value in values:
session.execute(SimpleStatement(
"insert into test (id, mytext, col1) values (1, '{}', null) ".format(
value
),
consistency_level=CL.ALL
))
failure_msg = ("Scanned over.* tombstones.* query aborted")
@timed(25)
def read_failure_query():
try:
session.execute(SimpleStatement("select * from test where id in (1,2,3,4,5)", consistency_level=CL.ALL))
except ReadFailure as exc:
if have_v5_protocol:
# at least one replica should have responded with a tombstone error
self.assertIsNotNone(exc.error_code_map)
self.assertEqual(0x0001, exc.error_code_map.values()[0])
except Exception:
raise
else:
self.fail('Expected ReadFailure')
read_failure_query()
# In almost all cases, we should find the failure message on node1 within a few seconds.
# If it is not on node1, we grep all logs, as it *absolutely* should be somewhere.
# If we still cannot find it then, we fail the test, as this is a problem.
try:
node1.watch_log_for(failure_msg, timeout=5)
except TimeoutError:
failure = (node1.grep_log(failure_msg) or
node2.grep_log(failure_msg) or
node3.grep_log(failure_msg))
self.assertTrue(failure, ("Cannot find tombstone failure threshold error in log "
"after failed query"))
mark1 = node1.mark_log()
mark2 = node2.mark_log()
mark3 = node3.mark_log()
@timed(35)
def range_request_failure_query():
try:
session.execute(SimpleStatement("select * from test", consistency_level=CL.ALL))
except ReadFailure as exc:
if have_v5_protocol:
# at least one replica should have responded with a tombstone error
self.assertIsNotNone(exc.error_code_map)
self.assertEqual(0x0001, exc.error_code_map.values()[0])
except Exception:
raise
else:
self.fail('Expected ReadFailure')
range_request_failure_query()
# In almost all cases, we should find the failure message on node1 within a few seconds.
# If it is not on node1, we grep all logs, as it *absolutely* should be somewhere.
# If we still cannot find it then, we fail the test, as this is a problem.
try:
node1.watch_log_for(failure_msg, from_mark=mark1, timeout=5)
except TimeoutError:
failure = (node1.grep_log(failure_msg, from_mark=mark1) or
node2.grep_log(failure_msg, from_mark=mark2) or
node3.grep_log(failure_msg, from_mark=mark3))
self.assertTrue(failure, ("Cannot find tombstone failure threshold error in log "
"after range_request_timeout_query"))
def short_read_test(self):
"""
@jira_ticket CASSANDRA-9460
"""
cluster = self.cluster
# Disable hinted handoff and set batch commit log so this doesn't
# interfer with the test
cluster.set_configuration_options(values={'hinted_handoff_enabled': False})
cluster.set_batch_commitlog(enabled=True)
cluster.populate(3).start(wait_other_notice=True)
node1, node2, node3 = cluster.nodelist()
session = self.patient_cql_connection(node1)
self.create_ks(session, 'ks', 3)
self.create_cf(session, 'cf', read_repair=0.0)
normal_key = 'normal'
reversed_key = 'reversed'
# Repeat this test 10 times to make it more easy to spot a null pointer exception caused by a race, see CASSANDRA-9460
for k in xrange(10):
# insert 9 columns in two rows
insert_columns(self, session, normal_key, 9)
insert_columns(self, session, reversed_key, 9)
# Delete 3 first columns (and 3 last columns, for the reversed version) with a different node dead each time
for node, column_number_to_delete in zip(range(1, 4), range(3)):
self.stop_node(node)
self.delete(node, normal_key, column_number_to_delete)
self.delete(node, reversed_key, 8 - column_number_to_delete)
self.restart_node(node)
# Query 3 firsts columns in normal order
session = self.patient_cql_connection(node1, 'ks')
query = SimpleStatement(
'SELECT c, v FROM cf WHERE key=\'k{}\' LIMIT 3'.format(normal_key),
consistency_level=ConsistencyLevel.QUORUM)
rows = list(session.execute(query))
res = rows
assert_length_equal(res, 3)
# value 0, 1 and 2 have been deleted
for i in xrange(1, 4):
self.assertEqual('value{}'.format(i + 2), res[i - 1][1])
# Query 3 firsts columns in reverse order
session = self.patient_cql_connection(node1, 'ks')
query = SimpleStatement(
'SELECT c, v FROM cf WHERE key=\'k{}\' ORDER BY c DESC LIMIT 3'.format(reversed_key),
consistency_level=ConsistencyLevel.QUORUM)
rows = list(session.execute(query))
res = rows
assert_length_equal(res, 3)
# value 6, 7 and 8 have been deleted
for i in xrange(0, 3):
self.assertEqual('value{}'.format(5 - i), res[i][1])
session.execute('TRUNCATE cf')
def alter_rf_and_run_read_repair_test(self):
"""
@jira_ticket CASSANDRA-10655
@jira_ticket CASSANDRA-10657
Test that querying only a subset of all the columns in a row doesn't confuse read-repair to avoid
the problem described in CASSANDRA-10655.
"""
session = self.patient_cql_connection(self.cluster.nodelist()[0])
session.execute("""CREATE KEYSPACE alter_rf_test
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};"""
)
session.execute(
"CREATE TABLE alter_rf_test.t1 (k int PRIMARY KEY, a int, b int);")
session.execute(
"INSERT INTO alter_rf_test.t1 (k, a, b) VALUES (1, 1, 1);")
cl_one_stmt = SimpleStatement(
"SELECT * FROM alter_rf_test.t1 WHERE k=1",
consistency_level=ConsistencyLevel.ONE)
# identify the initial replica and trigger a flush to ensure reads come from sstables
initial_replica, non_replicas = self.identify_initial_placement(
'alter_rf_test', 't1', 1)
debug("At RF=1 replica for data is " + initial_replica.name)
initial_replica.flush()
# At RF=1, it shouldn't matter which node we query, as the actual data should always come from the
# initial replica when reading at CL ONE
for n in self.cluster.nodelist():
debug("Checking " + n.name)
session = self.patient_exclusive_cql_connection(n)
assert_one(session,
"SELECT * FROM alter_rf_test.t1 WHERE k=1", [1, 1, 1],
cl=ConsistencyLevel.ONE)
# Alter so RF=n but don't repair, then execute a query which selects only a subset of the columns. Run this at
# CL ALL on one of the nodes which doesn't currently have the data, triggering a read repair.
# The expectation will be that every replicas will have been repaired for that column (but we make no assumptions
# on the other columns).
debug("Changing RF from 1 to 3")
session.execute("""ALTER KEYSPACE alter_rf_test
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};"""
)
cl_all_stmt = SimpleStatement(
"SELECT a FROM alter_rf_test.t1 WHERE k=1",
consistency_level=ConsistencyLevel.ALL)
debug(
"Executing SELECT on non-initial replica to trigger read repair " +
non_replicas[0].name)
read_repair_session = self.patient_exclusive_cql_connection(
non_replicas[0])
# result of the CL ALL query contains only the selected column
assert_one(read_repair_session,
"SELECT a FROM alter_rf_test.t1 WHERE k=1", [1],
cl=ConsistencyLevel.ALL)
# Check the results of the read repair by querying each replica again at CL ONE
debug("Re-running SELECTs at CL ONE to verify read repair")
for n in self.cluster.nodelist():
debug("Checking " + n.name)
session = self.patient_exclusive_cql_connection(n)
res = rows_to_list(session.execute(cl_one_stmt))
# Column a must be 1 everywhere, and column b must be either 1 or None everywhere
self.assertIn(res[0][:2], [[1, 1], [1, None]])
# Now query at ALL but selecting all columns
query = "SELECT * FROM alter_rf_test.t1 WHERE k=1"
debug(
"Executing SELECT on non-initial replica to trigger read repair " +
non_replicas[0].name)
read_repair_session = self.patient_exclusive_cql_connection(
non_replicas[0])
assert_one(session, query, [1, 1, 1], cl=ConsistencyLevel.ALL)
# Check all replica is fully up to date
debug("Re-running SELECTs at CL ONE to verify read repair")
for n in self.cluster.nodelist():
debug("Checking " + n.name)
session = self.patient_exclusive_cql_connection(n)
assert_one(session, query, [1, 1, 1], cl=ConsistencyLevel.ONE)
def execute(query, params=None, consistency_level=None):
params = params or {}
consistency_level = get_consistency_level(consistency_level)
session = get_connection_pool()
query = SimpleStatement(query, consistency_level=consistency_level)
return session.execute(query, parameters=params)
def upload_file(self):
"""
This will upload all the selected csv/xml files in the database for the given algorithm.
:return: Json Response
"""
try:
assert self._db_connection, {
STATUS_KEY: HTTP_500_INTERNAL_SERVER_ERROR,
MESSAGE_KEY: DB_ERROR
}
batch = BatchStatement()
error = False
error_message = None
FLAG = 0
algo_tag = "TMT"
"""if not self.files:
error = True
error_message = "No files to upload"
file_names_list = self.file_names
# select_query = SELECT_ALGORITHM_NAME_QUERY.format(self.algo_name, ",".join(map(lambda x: "'" + x + "'", file_names_list)))
select_query = "SELECT algorithm_name from csv.configuration_details WHERE algorithm_name = '{}' ALLOW FILTERING".format(
self.algo_name)
result_set = self._csql_session.execute(select_query)
if result_set[0]['count'] == 0 or result_set[0]['count'] < len(file_names_list):
error_message = "Please give the existing algorithm or file name"
return JsonResponse({MESSAGE_KEY: error_message}, status=HTTP_500_INTERNAL_SERVER_ERROR)"""
for file in self.files:
"""if file.name not in self.file_names:
error = True
error_message = "Uploaded file name("+file.name+") not found in given file name list"
break"""
description = None
if file.name in self.description:
description = self.description[file.name]
# print(self.description[f.name])
LAST_MODIFIED_DATE = str(round(time.time() * 1000))
#file_data = str(file.read(), 'utf-8')
extension = os.path.splitext(file.name)[1]
json_data = ""
if extension == ".csv":
file_data = pandas.read_csv(file, encoding='ISO-8859-1')
json_data = file_data.to_json()
elif extension == ".xml":
file_data = et.parse(file)
xml_str = ElementTree.tostring(file_data.getroot(),
encoding='unicode')
json_data = json.dumps(xmltodict.parse(xml_str))
elif extension == ".joblib":
#json_data = joblib.load((os.path.realpath(file.name())))
#print(json_data)
#print(file.)
json_datas = joblib.load(file)
# json_datas = pickle.load(open(file, "rb"))
print(json_datas)
json_data = escape(str(json_datas))
print(json_data)
# json_data = json.dumps(file)
# json_datas = joblib.load(file)
# json_datas = pickle.load(open(file, "rb"))
# print(json_datas)
# json_data = escape(str(json_datas))
# print(json_data)
# json_data = file
#print(json_data)
#print("ggggggg")
#json_data = file.read().decode()
# insert query into cassandra table
# insert_query = """ INSERT into csv.configuration_details (algorithm_name, file_param_name, description, """ \
# """file_content, last_modified_date, type, flag) values ('{}', '{}', '{}', {}, {}, 2, {})""".format(
# self.algo_name,
# file.name,
# description,
# "textAsBlob('" + json_data + "')",
# LAST_MODIFIED_DATE,
# FLAG)
insert_query = """ INSERT into {}.{} (algorithm_name, file_param_name, description, """ \
"""file_content, last_modified_date, type, flag, algo_tag) values ('{}', '{}', '{}', {}, {}, 2, {}, '{}')""".format(NAME, TABLE_NAME,
self.algo_name,
file.name,
description,
"textAsBlob('" + json_data + "')",
LAST_MODIFIED_DATE,
FLAG,
algo_tag)
# batch insert initialisation
batch.add(SimpleStatement(insert_query))
if error is True:
return JsonResponse({MESSAGE_KEY: error_message},
status=HTTP_500_INTERNAL_SERVER_ERROR)
self._csql_session.execute(batch, timeout=200.0)
return JsonResponse({MESSAGE_KEY: UPLOADED_SUCCESSFULLY},
safe=False)
except AssertionError as e:
print(e)
log_error(e)
return JsonResponse({MESSAGE_KEY: e.args[0][MESSAGE_KEY]},
status=e.args[0][STATUS_KEY])
except Exception as e:
print("Error while inserting data - " + str(e))
log_error(traceback.format_exc())
return JsonResponse(
{MESSAGE_KEY: EXCEPTION_CAUSE.format(traceback.format_exc())},
status=HTTP_500_INTERNAL_SERVER_ERROR)
def remote_query_test(self):
"""
Check that a query running on a node other than the coordinator times out:
- populate the cluster with 2 nodes
- set a 1-second read timeout
- start one node without having it join the ring
- start the other node with read_iteration_delay set to 1.5 seconds
- (this will cause read queries to take longer than the read timeout)
- CREATE a table
- INSERT 5000 rows on a session on the node that is not a member of the ring
- run SELECT statements and assert they fail
# TODO refactor SELECT statements:
# - run the statements in a loop to reduce duplication
# - watch the log after each query
# - assert we raise the right error
"""
cluster = self.cluster
cluster.set_configuration_options(
values={'read_request_timeout_in_ms': 1000})
cluster.populate(2)
node1, node2 = cluster.nodelist()
node1.start(wait_for_binary_proto=True,
join_ring=False) # ensure other node executes queries
node2.start(wait_for_binary_proto=True,
jvm_args=[
"-Dcassandra.monitoring_check_interval_ms=50",
"-Dcassandra.test.read_iteration_delay_ms=1500"
]) # see above for explanation
session = self.patient_exclusive_cql_connection(node1)
self.create_ks(session, 'ks', 1)
session.execute("""
CREATE TABLE test2 (
id int,
col int,
val text,
PRIMARY KEY(id, col)
);
""")
for i, j in itertools.product(range(500), range(10)):
session.execute(
"INSERT INTO test2 (id, col, val) VALUES ({}, {}, 'foo')".
format(i, j))
mark = node2.mark_log()
statement = SimpleStatement("SELECT * from test2",
consistency_level=ConsistencyLevel.ONE,
retry_policy=FallthroughRetryPolicy())
assert_unavailable(lambda c: debug(c.execute(statement)), session)
statement = SimpleStatement("SELECT * from test2 where id = 1",
consistency_level=ConsistencyLevel.ONE,
retry_policy=FallthroughRetryPolicy())
assert_unavailable(lambda c: debug(c.execute(statement)), session)
statement = SimpleStatement(
"SELECT * from test2 where id IN (1, 10, 20) AND col < 10",
consistency_level=ConsistencyLevel.ONE,
retry_policy=FallthroughRetryPolicy())
assert_unavailable(lambda c: debug(c.execute(statement)), session)
statement = SimpleStatement(
"SELECT * from test2 where col > 5 ALLOW FILTERING",
consistency_level=ConsistencyLevel.ONE,
retry_policy=FallthroughRetryPolicy())
assert_unavailable(lambda c: debug(c.execute(statement)), session)
node2.watch_log_for("operations timed out", from_mark=mark, timeout=60)
