forked from apache/cassandra-dtest
-
Notifications
You must be signed in to change notification settings - Fork 0
/
scrub_test.py
473 lines (372 loc) · 18.5 KB
/
scrub_test.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
import glob
import os
import re
import subprocess
import time
import uuid
import pytest
import parse
import logging
from ccmlib import common
from dtest import Tester, create_ks, create_cf
from tools.assertions import assert_length_equal, assert_stderr_clean
since = pytest.mark.since
logger = logging.getLogger(__name__)
KEYSPACE = 'ks'
class TestHelper(Tester):
@pytest.fixture(scope='function', autouse=True)
def fixture_set_cluster_settings(self, fixture_dtest_setup):
"""
disable JBOD configuration for scrub tests.
range-aware JBOD can skip generation in SSTable,
and some tests rely on generation numbers/
(see CASSANDRA-11693 and increase_sstable_generations)
"""
fixture_dtest_setup.cluster.set_datadir_count(1)
def get_table_paths(self, table):
"""
Return the path where the table sstables are located
"""
node1 = self.cluster.nodelist()[0]
paths = []
for data_dir in node1.data_directories():
basepath = os.path.join(data_dir, KEYSPACE)
for x in os.listdir(basepath):
if x.startswith(table):
paths.append(os.path.join(basepath, x))
break
return paths
def get_index_paths(self, table, index):
"""
Return the paths where the index sstables are located
"""
paths = self.get_table_paths(table)
index_paths = []
for path in paths:
index_paths.append(os.path.join(path, '.' + index))
return paths
def get_sstable_files(self, paths):
"""
Return the sstable files at a specific location
"""
ret = []
logger.debug('Checking sstables in {}'.format(paths))
for ext in ('*.db', '*.txt', '*.adler32', '*.sha1'):
for path in paths:
for fname in glob.glob(os.path.join(path, ext)):
bname = os.path.basename(fname)
logger.debug('Found sstable file {}'.format(bname))
ret.append(bname)
return ret
def delete_non_essential_sstable_files(self, table):
"""
Delete all sstable files except for the -Data.db file and the
-Statistics.db file (only available in >= 3.0)
"""
for fname in self.get_sstable_files(self.get_table_paths(table)):
if not fname.endswith("-Data.db") and not fname.endswith("-Statistics.db"):
paths = self.get_table_paths(table)
for path in paths:
fullname = os.path.join(path, fname)
if (os.path.exists(fullname)):
logger.debug('Deleting {}'.format(fullname))
os.remove(fullname)
def get_sstables(self, table, indexes):
"""
Return the sstables for a table and the specified indexes of this table
"""
sstables = {}
table_sstables = self.get_sstable_files(self.get_table_paths(table))
assert len(table_sstables) > 0
sstables[table] = sorted(table_sstables)
for index in indexes:
index_sstables = self.get_sstable_files(self.get_index_paths(table, index))
assert len(index_sstables) > 0
sstables[index] = sorted('{}/{}'.format(index, sstable) for sstable in index_sstables)
return sstables
def launch_nodetool_cmd(self, cmd):
"""
Launch a nodetool command and check the result is empty (no error)
"""
node1 = self.cluster.nodelist()[0]
response = node1.nodetool(cmd).stdout
if not common.is_win(): # nodetool always prints out on windows
assert_length_equal(response, 0) # nodetool does not print anything unless there is an error
def launch_standalone_scrub(self, ks, cf, reinsert_overflowed_ttl=False, no_validate=False):
"""
Launch the standalone scrub
"""
node1 = self.cluster.nodelist()[0]
env = common.make_cassandra_env(node1.get_install_cassandra_root(), node1.get_node_cassandra_root())
scrub_bin = node1.get_tool('sstablescrub')
logger.debug(scrub_bin)
args = [scrub_bin]
if reinsert_overflowed_ttl:
args += ['--reinsert-overflowed-ttl']
if no_validate:
args += ['--no-validate']
args += [ks, cf] if reinsert_overflowed_ttl else [ks, cf]
p = subprocess.Popen(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
logger.debug(out.decode("utf-8"))
# if we have less than 64G free space, we get this warning - ignore it
if err and "Consider adding more capacity" not in err.decode("utf-8"):
logger.debug(err.decode("utf-8"))
assert_stderr_clean(err.decode("utf-8"))
def perform_node_tool_cmd(self, cmd, table, indexes):
"""
Perform a nodetool command on a table and the indexes specified
"""
self.launch_nodetool_cmd('{} {} {}'.format(cmd, KEYSPACE, table))
for index in indexes:
self.launch_nodetool_cmd('{} {} {}.{}'.format(cmd, KEYSPACE, table, index))
def flush(self, table, *indexes):
"""
Flush table and indexes via nodetool, and then return all sstables
in a dict keyed by the table or index name.
"""
self.perform_node_tool_cmd('flush', table, indexes)
return self.get_sstables(table, indexes)
def scrub(self, table, *indexes):
"""
Scrub table and indexes via nodetool, and then return all sstables
in a dict keyed by the table or index name.
"""
self.perform_node_tool_cmd('scrub', table, indexes)
time.sleep(.1)
return self.get_sstables(table, indexes)
def standalonescrub(self, table, *indexes):
"""
Launch standalone scrub on table and indexes, and then return all sstables
in a dict keyed by the table or index name.
"""
self.launch_standalone_scrub(KEYSPACE, table)
for index in indexes:
self.launch_standalone_scrub(KEYSPACE, '{}.{}'.format(table, index))
return self.get_sstables(table, indexes)
def increment_generation_by(self, sstable, generation_increment):
"""
Set the generation number for an sstable file name
"""
return re.sub('(\d(?!\d))\-', lambda x: str(int(x.group(1)) + generation_increment) + '-', sstable)
def increase_sstable_generations(self, sstables):
"""
After finding the number of existing sstables, increase all of the
generations by that amount.
"""
for table_or_index, table_sstables in list(sstables.items()):
increment_by = len(set(parse.search('{}-{increment_by}-{suffix}.{file_extention}', s).named['increment_by'] for s in table_sstables))
sstables[table_or_index] = [self.increment_generation_by(s, increment_by) for s in table_sstables]
logger.debug('sstables after increment {}'.format(str(sstables)))
@since('2.2')
class TestScrubIndexes(TestHelper):
"""
Test that we scrub indexes as well as their parent tables
"""
def create_users(self, session):
columns = {"password": "varchar", "gender": "varchar", "session_token": "varchar", "state": "varchar", "birth_year": "bigint"}
create_cf(session, 'users', columns=columns)
session.execute("CREATE INDEX gender_idx ON users (gender)")
session.execute("CREATE INDEX state_idx ON users (state)")
session.execute("CREATE INDEX birth_year_idx ON users (birth_year)")
def update_users(self, session):
session.execute("INSERT INTO users (KEY, password, gender, state, birth_year) VALUES ('user1', 'ch@ngem3a', 'f', 'TX', 1978)")
session.execute("INSERT INTO users (KEY, password, gender, state, birth_year) VALUES ('user2', 'ch@ngem3b', 'm', 'CA', 1982)")
session.execute("INSERT INTO users (KEY, password, gender, state, birth_year) VALUES ('user3', 'ch@ngem3c', 'f', 'TX', 1978)")
session.execute("INSERT INTO users (KEY, password, gender, state, birth_year) VALUES ('user4', 'ch@ngem3d', 'm', 'CA', 1982)")
session.execute("INSERT INTO users (KEY, password, gender, state, birth_year) VALUES ('user5', 'ch@ngem3e', 'f', 'TX', 1978)")
session.execute("INSERT INTO users (KEY, password, gender, state, birth_year) VALUES ('user6', 'ch@ngem3f', 'm', 'CA', 1982)")
session.execute("INSERT INTO users (KEY, password, gender, state, birth_year) VALUES ('user7', 'ch@ngem3g', 'f', 'TX', 1978)")
session.execute("INSERT INTO users (KEY, password, gender, state, birth_year) VALUES ('user8', 'ch@ngem3h', 'm', 'CA', 1982)")
session.execute("DELETE FROM users where KEY = 'user1'")
session.execute("DELETE FROM users where KEY = 'user5'")
session.execute("DELETE FROM users where KEY = 'user7'")
def query_users(self, session):
ret = list(session.execute("SELECT * FROM users"))
ret.extend(list(session.execute("SELECT * FROM users WHERE state='TX'")))
ret.extend(list(session.execute("SELECT * FROM users WHERE gender='f'")))
ret.extend(list(session.execute("SELECT * FROM users WHERE birth_year=1978")))
assert_length_equal(ret, 8)
return ret
def test_scrub_static_table(self):
cluster = self.cluster
cluster.populate(1).start()
node1 = cluster.nodelist()[0]
session = self.patient_cql_connection(node1)
create_ks(session, KEYSPACE, 1)
self.create_users(session)
self.update_users(session)
initial_users = self.query_users(session)
initial_sstables = self.flush('users', 'gender_idx', 'state_idx', 'birth_year_idx')
scrubbed_sstables = self.scrub('users', 'gender_idx', 'state_idx', 'birth_year_idx')
self.increase_sstable_generations(initial_sstables)
assert initial_sstables == scrubbed_sstables
users = self.query_users(session)
assert initial_users == users
# Scrub and check sstables and data again
scrubbed_sstables = self.scrub('users', 'gender_idx', 'state_idx', 'birth_year_idx')
self.increase_sstable_generations(initial_sstables)
assert initial_sstables == scrubbed_sstables
users = self.query_users(session)
assert initial_users == users
# Restart and check data again
cluster.stop()
cluster.start()
session = self.patient_cql_connection(node1)
session.execute('USE {}'.format(KEYSPACE))
users = self.query_users(session)
assert initial_users == users
def test_standalone_scrub(self):
cluster = self.cluster
cluster.populate(1).start()
node1 = cluster.nodelist()[0]
session = self.patient_cql_connection(node1)
create_ks(session, KEYSPACE, 1)
self.create_users(session)
self.update_users(session)
initial_users = self.query_users(session)
initial_sstables = self.flush('users', 'gender_idx', 'state_idx', 'birth_year_idx')
cluster.stop()
scrubbed_sstables = self.standalonescrub('users', 'gender_idx', 'state_idx', 'birth_year_idx')
self.increase_sstable_generations(initial_sstables)
assert initial_sstables == scrubbed_sstables
cluster.start()
session = self.patient_cql_connection(node1)
session.execute('USE {}'.format(KEYSPACE))
users = self.query_users(session)
assert initial_users == users
def test_scrub_collections_table(self):
cluster = self.cluster
cluster.populate(1).start()
node1 = cluster.nodelist()[0]
session = self.patient_cql_connection(node1)
create_ks(session, KEYSPACE, 1)
session.execute("CREATE TABLE users (user_id uuid PRIMARY KEY, email text, uuids list<uuid>)")
session.execute("CREATE INDEX user_uuids_idx on users (uuids)")
_id = uuid.uuid4()
num_users = 100
for i in range(0, num_users):
user_uuid = uuid.uuid4()
session.execute(("INSERT INTO users (user_id, email) values ({user_id}, 'test@example.com')").format(user_id=user_uuid))
session.execute(("UPDATE users set uuids = [{id}] where user_id = {user_id}").format(id=_id, user_id=user_uuid))
initial_users = list(session.execute(("SELECT * from users where uuids contains {some_uuid}").format(some_uuid=_id)))
assert num_users == len(initial_users)
initial_sstables = self.flush('users', 'user_uuids_idx')
scrubbed_sstables = self.scrub('users', 'user_uuids_idx')
self.increase_sstable_generations(initial_sstables)
assert initial_sstables == scrubbed_sstables
users = list(session.execute(("SELECT * from users where uuids contains {some_uuid}").format(some_uuid=_id)))
assert initial_users == users
scrubbed_sstables = self.scrub('users', 'user_uuids_idx')
self.increase_sstable_generations(initial_sstables)
assert initial_sstables == scrubbed_sstables
users = list(session.execute(("SELECT * from users where uuids contains {some_uuid}").format(some_uuid=_id)))
assert initial_users == users
class TestScrub(TestHelper):
"""
Generic tests for scrubbing
"""
def create_users(self, session):
columns = {"password": "varchar", "gender": "varchar", "session_token": "varchar", "state": "varchar", "birth_year": "bigint"}
create_cf(session, 'users', columns=columns)
def update_users(self, session):
session.execute("INSERT INTO users (KEY, password, gender, state, birth_year) VALUES ('user1', 'ch@ngem3a', 'f', 'TX', 1978)")
session.execute("INSERT INTO users (KEY, password, gender, state, birth_year) VALUES ('user2', 'ch@ngem3b', 'm', 'CA', 1982)")
session.execute("INSERT INTO users (KEY, password, gender, state, birth_year) VALUES ('user3', 'ch@ngem3c', 'f', 'TX', 1978)")
session.execute("INSERT INTO users (KEY, password, gender, state, birth_year) VALUES ('user4', 'ch@ngem3d', 'm', 'CA', 1982)")
session.execute("INSERT INTO users (KEY, password, gender, state, birth_year) VALUES ('user5', 'ch@ngem3e', 'f', 'TX', 1978)")
session.execute("INSERT INTO users (KEY, password, gender, state, birth_year) VALUES ('user6', 'ch@ngem3f', 'm', 'CA', 1982)")
session.execute("INSERT INTO users (KEY, password, gender, state, birth_year) VALUES ('user7', 'ch@ngem3g', 'f', 'TX', 1978)")
session.execute("INSERT INTO users (KEY, password, gender, state, birth_year) VALUES ('user8', 'ch@ngem3h', 'm', 'CA', 1982)")
session.execute("DELETE FROM users where KEY = 'user1'")
session.execute("DELETE FROM users where KEY = 'user5'")
session.execute("DELETE FROM users where KEY = 'user7'")
def query_users(self, session):
ret = list(session.execute("SELECT * FROM users"))
assert_length_equal(ret, 5)
return ret
def test_nodetool_scrub(self):
cluster = self.cluster
cluster.populate(1).start()
node1 = cluster.nodelist()[0]
# we don't want automatic minor compaction because we want to block on
# compactions
node1.nodetool('disableautocompaction')
session = self.patient_cql_connection(node1)
create_ks(session, KEYSPACE, 1)
self.create_users(session)
self.update_users(session)
initial_users = self.query_users(session)
initial_sstables = self.flush('users')
scrubbed_sstables = self.scrub('users')
self.increase_sstable_generations(initial_sstables)
assert initial_sstables == scrubbed_sstables
users = self.query_users(session)
assert initial_users == users
# Scrub and check sstables and data again
scrubbed_sstables = self.scrub('users')
self.increase_sstable_generations(initial_sstables)
assert initial_sstables == scrubbed_sstables
users = self.query_users(session)
assert initial_users == users
# Restart and check data again
cluster.stop()
cluster.start()
session = self.patient_cql_connection(node1)
session.execute('USE {}'.format(KEYSPACE))
users = self.query_users(session)
assert initial_users == users
def test_standalone_scrub(self):
cluster = self.cluster
cluster.populate(1).start()
node1 = cluster.nodelist()[0]
session = self.patient_cql_connection(node1)
create_ks(session, KEYSPACE, 1)
self.create_users(session)
self.update_users(session)
initial_users = self.query_users(session)
initial_sstables = self.flush('users')
cluster.stop()
scrubbed_sstables = self.standalonescrub('users')
self.increase_sstable_generations(initial_sstables)
assert initial_sstables == scrubbed_sstables
cluster.start()
session = self.patient_cql_connection(node1)
session.execute('USE {}'.format(KEYSPACE))
users = self.query_users(session)
assert initial_users == users
def test_standalone_scrub_essential_files_only(self):
cluster = self.cluster
cluster.populate(1).start()
node1 = cluster.nodelist()[0]
session = self.patient_cql_connection(node1)
create_ks(session, KEYSPACE, 1)
self.create_users(session)
self.update_users(session)
initial_users = self.query_users(session)
initial_sstables = self.flush('users')
cluster.stop()
self.delete_non_essential_sstable_files('users')
scrubbed_sstables = self.standalonescrub('users')
self.increase_sstable_generations(initial_sstables)
assert initial_sstables == scrubbed_sstables
cluster.start()
session = self.patient_cql_connection(node1)
session.execute('USE {}'.format(KEYSPACE))
users = self.query_users(session)
assert initial_users == users
def test_scrub_with_UDT(self):
"""
@jira_ticket CASSANDRA-7665
"""
cluster = self.cluster
cluster.populate(1).start()
node1 = cluster.nodelist()[0]
session = self.patient_cql_connection(node1)
session.execute("CREATE KEYSPACE test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1 };")
session.execute("use test;")
session.execute("CREATE TYPE point_t (x double, y double);")
node1.nodetool("scrub")
time.sleep(2)
match = node1.grep_log("org.apache.cassandra.serializers.MarshalException: Not enough bytes to read a set")
assert len(match) == 0