def test_no_corruption_restore_writemeta_append(library, library_name):
large_ts = create_test_data(size=2000,
cols=100,
index=True,
multiindex=False,
random_data=True,
random_ids=True)
rows_per_append = 100
last_row = n_append(library, library_name, 9, rows_per_append, large_ts, 0)
library.write_metadata(symbol, metadata={'abc': 'xyz'})
n_append(library, library_name, 9, rows_per_append, large_ts, last_row)
library.write_metadata(symbol, metadata={'abc2': 'xyz2'})
# Corrupts all versions between the version that row "restore_from_row" was written,
restore_from_row = rows_per_append * 10
library.restore_version(symbol, 'snap_{}'.format(restore_from_row))
library.write_metadata(symbol, metadata={'abc3': 'xyz3'})
library.append(symbol, large_ts[restore_from_row:restore_from_row + 50])
library.write_metadata(symbol, metadata={'abc4': 'xyz4'})
last_v = library._versions.find_one(sort=[('version', pymongo.DESCENDING)])
vsu.analyze_symbol(library, symbol, 0, last_v['version'] + 1)
# Verify no versions have been corrupted
for v in library._versions.find(sort=[('version', pymongo.DESCENDING)]):
library.read(symbol, as_of=v['version'])
def test_append_fail_after_delete_noupsert(library, library_name):
large_ts = create_test_data(size=2000,
cols=100,
index=True,
multiindex=False,
random_data=True,
random_ids=True)
library.write(symbol, large_ts[0:1000]) #v1
library.snapshot('snap_a')
library.append(symbol, large_ts[1000:1010]) #v2
library.snapshot('snap_b')
library.append(symbol, large_ts[1010:1020]) #v3
library.snapshot('snap_c')
library.append(symbol, large_ts[1030:1040]) #v4
library.delete(symbol) #v5
library.append(symbol, large_ts[1040:1050], upsert=False) # v6
last_v = library._versions.find_one(sort=[('version', pymongo.DESCENDING)])
vsu.analyze_symbol(library, symbol, 0, last_v['version'] + 1)
# Verify no versions have been corrupted
for v in library._versions.find(sort=[('version', pymongo.DESCENDING)]):
library.read(symbol, as_of=v['version'])
def test_no_corruption_restore_writemeta_append(library, library_name):
large_ts = create_test_data(size=2000, cols=100,
index=True, multiindex=False,
random_data=True, random_ids=True)
rows_per_append = 100
last_row = n_append(library, library_name, 9, rows_per_append, large_ts, 0)
library.write_metadata(symbol, metadata={'abc': 'xyz'})
n_append(library, library_name, 9, rows_per_append, large_ts, last_row)
library.write_metadata(symbol, metadata={'abc2': 'xyz2'})
# Corrupts all versions between the version that row "restore_from_row" was written,
restore_from_row = rows_per_append * 10
library.restore_version(symbol, 'snap_{}'.format(restore_from_row))
library.write_metadata(symbol, metadata={'abc3': 'xyz3'})
library.append(symbol, large_ts[restore_from_row:restore_from_row + 50])
library.write_metadata(symbol, metadata={'abc4': 'xyz4'})
last_v = library._versions.find_one(sort=[('version', pymongo.DESCENDING)])
vsu.analyze_symbol(library, symbol, 0, last_v['version'] + 1)
# Verify no versions have been corrupted
for v in library._versions.find(sort=[('version', pymongo.DESCENDING)]):
library.read(symbol, as_of=v['version'])
def _corrupt_with_append_only(library, library_name):
def do_fail(version):
raise Exception('test')
large_ts = create_test_data(size=2000, cols=100,
index=True, multiindex=False,
random_data=True, random_ids=True)
library.write(symbol, large_ts[0:1000]) # v1
library.snapshot('snap_write_a')
library.append(symbol, large_ts[1000:1010]) # v2
library.snapshot('snap_write_b')
# Here we simulate a scenario where an append succeeds to insert the data segments,
# but fails to insert the version document (i.e. Mongo error occurred)
orig_insert_version = library._insert_version
library._insert_version = do_fail
try:
library.append(symbol, large_ts[1010:1020]) # v3
except:
pass
library._insert_version = orig_insert_version
library.write_metadata(symbol, {'hello': 'there'}) # , prune_previous_version=False)
# Appending subsequently overlapping and non-SHA-matching data cause data corruption
library.append(symbol, large_ts[1018:1030]) # , prune_previous_version=False)
last_v = library._versions.find_one(sort=[('version', pymongo.DESCENDING)])
vsu.analyze_symbol(library, symbol, 0, last_v['version'] + 1)
# Verify no versions have been corrupted
for v in library._versions.find(sort=[('version', pymongo.DESCENDING)]):
library.read(symbol, as_of=v['version'])
def test_no_corruption_restore_append_non_overlapping_tstamps(library, library_name):
large_ts = create_test_data(size=2000, cols=100,
index=True, multiindex=False,
random_data=True, random_ids=True)
# Append with 50 small uncompressed segments (no new base yet)
last_row_b = n_append(library, library_name, 50, 25, large_ts, 0, False, True)
library.snapshot('snap_A')
# Append with 20 more small segments, causes once copy-rewrite with new base, and then some small appended segments
n_append(library, library_name, 15, 25, large_ts, last_row_b, True, True)
library.restore_version(symbol, as_of='snap_A')
last_row = n_append(library, library_name, 1, 40, large_ts, last_row_b, False, True)
library.snapshot('snap_B')
# Corrupts all versions
last_row = n_append(library, library_name, 1, 10, large_ts, last_row, False, True)
last_row = n_append(library, library_name, 8, 20, large_ts, last_row, False, True)
library.snapshot('snap_C')
last_v = library._versions.find_one(sort=[('version', pymongo.DESCENDING)])
vsu.analyze_symbol(library, symbol, 0, last_v['version'] + 1)
# Verify no versions have been corrupted
for v in library._versions.find(sort=[('version', pymongo.DESCENDING)]):
library.read(symbol, as_of=v['version'])
def test_restore_append_overlapping_corrupts_last(library, library_name):
large_ts = create_test_data(size=2000, cols=100,
index=True, multiindex=False,
random_data=True, random_ids=True)
library.write(symbol, large_ts[0:1000])
library.snapshot('snap_write_a')
library.append(symbol, large_ts[1000:1010])
library.restore_version(symbol, as_of='snap_write_a', prune_previous_version=True)
library.append(symbol, large_ts[1000:1012])
last_v = library._versions.find_one(sort=[('version', pymongo.DESCENDING)])
vsu.analyze_symbol(library, symbol, 0, last_v['version'] + 1)
# Verify no versions have been corrupted
for v in library._versions.find(sort=[('version', pymongo.DESCENDING)]):
library.read(symbol, as_of=v['version'])
def _corrupt_with_append_only(library, library_name):
def do_fail(version):
raise Exception('test')
large_ts = create_test_data(size=2000,
cols=100,
index=True,
multiindex=False,
random_data=True,
random_ids=True)
library.write(symbol, large_ts[0:1000]) # v1
library.snapshot('snap_write_a')
library.append(symbol, large_ts[1000:1010]) # v2
library.snapshot('snap_write_b')
# Here we simulate a scenario where an append succeeds to insert the data segments,
# but fails to insert the version document (i.e. Mongo error occurred)
orig_insert_version = library._insert_version
library._insert_version = do_fail
try:
library.append(symbol, large_ts[1010:1020]) # v3
except:
pass
library._insert_version = orig_insert_version
library.write_metadata(
symbol, {'hello': 'there'}) # , prune_previous_version=False)
# Appending subsequently overlapping and non-SHA-matching data cause data corruption
library.append(symbol,
large_ts[1018:1030]) # , prune_previous_version=False)
last_v = library._versions.find_one(sort=[('version', pymongo.DESCENDING)])
vsu.analyze_symbol(library, symbol, 0, last_v['version'] + 1)
# Verify no versions have been corrupted
for v in library._versions.find(sort=[('version', pymongo.DESCENDING)]):
library.read(symbol, as_of=v['version'])
def test_restore_append_overlapping_corrupts_last(library, library_name):
large_ts = create_test_data(size=2000,
cols=100,
index=True,
multiindex=False,
random_data=True,
random_ids=True)
library.write(symbol, large_ts[0:1000])
library.snapshot('snap_write_a')
library.append(symbol, large_ts[1000:1010])
library.restore_version(symbol,
as_of='snap_write_a',
prune_previous_version=True)
library.append(symbol, large_ts[1000:1012])
last_v = library._versions.find_one(sort=[('version', pymongo.DESCENDING)])
vsu.analyze_symbol(library, symbol, 0, last_v['version'] + 1)
# Verify no versions have been corrupted
for v in library._versions.find(sort=[('version', pymongo.DESCENDING)]):
library.read(symbol, as_of=v['version'])
def test_append_fail_after_delete_noupsert(library, library_name):
large_ts = create_test_data(size=2000, cols=100,
index=True, multiindex=False,
random_data=True, random_ids=True)
library.write(symbol, large_ts[0:1000]) #v1
library.snapshot('snap_a')
library.append(symbol, large_ts[1000:1010]) #v2
library.snapshot('snap_b')
library.append(symbol, large_ts[1010:1020]) #v3
library.snapshot('snap_c')
library.append(symbol, large_ts[1030:1040]) #v4
library.delete(symbol) #v5
library.append(symbol, large_ts[1040:1050], upsert=False) # v6
last_v = library._versions.find_one(sort=[('version', pymongo.DESCENDING)])
vsu.analyze_symbol(library, symbol, 0, last_v['version'] + 1)
# Verify no versions have been corrupted
for v in library._versions.find(sort=[('version', pymongo.DESCENDING)]):
library.read(symbol, as_of=v['version'])
def test_no_corruption_restore_append_non_overlapping_tstamps(
library, library_name):
large_ts = create_test_data(size=2000,
cols=100,
index=True,
multiindex=False,
random_data=True,
random_ids=True)
# Append with 50 small uncompressed segments (no new base yet)
last_row_b = n_append(library, library_name, 50, 25, large_ts, 0, False,
True)
library.snapshot('snap_A')
# Append with 20 more small segments, causes once copy-rewrite with new base, and then some small appended segments
n_append(library, library_name, 15, 25, large_ts, last_row_b, True, True)
library.restore_version(symbol, as_of='snap_A')
last_row = n_append(library, library_name, 1, 40, large_ts, last_row_b,
False, True)
library.snapshot('snap_B')
# Corrupts all versions
last_row = n_append(library, library_name, 1, 10, large_ts, last_row,
False, True)
last_row = n_append(library, library_name, 8, 20, large_ts, last_row,
False, True)
library.snapshot('snap_C')
last_v = library._versions.find_one(sort=[('version', pymongo.DESCENDING)])
vsu.analyze_symbol(library, symbol, 0, last_v['version'] + 1)
# Verify no versions have been corrupted
for v in library._versions.find(sort=[('version', pymongo.DESCENDING)]):
library.read(symbol, as_of=v['version'])
