def test_no_corruption_restore_writemeta_append(library, library_name): large_ts = create_test_data(size=2000, cols=100, index=True, multiindex=False, random_data=True, random_ids=True) rows_per_append = 100 last_row = n_append(library, library_name, 9, rows_per_append, large_ts, 0) library.write_metadata(symbol, metadata={'abc': 'xyz'}) n_append(library, library_name, 9, rows_per_append, large_ts, last_row) library.write_metadata(symbol, metadata={'abc2': 'xyz2'}) # Corrupts all versions between the version that row "restore_from_row" was written, restore_from_row = rows_per_append * 10 library.restore_version(symbol, 'snap_{}'.format(restore_from_row)) library.write_metadata(symbol, metadata={'abc3': 'xyz3'}) library.append(symbol, large_ts[restore_from_row:restore_from_row + 50]) library.write_metadata(symbol, metadata={'abc4': 'xyz4'}) last_v = library._versions.find_one(sort=[('version', pymongo.DESCENDING)]) vsu.analyze_symbol(library, symbol, 0, last_v['version'] + 1) # Verify no versions have been corrupted for v in library._versions.find(sort=[('version', pymongo.DESCENDING)]): library.read(symbol, as_of=v['version'])
def test_append_fail_after_delete_noupsert(library, library_name): large_ts = create_test_data(size=2000, cols=100, index=True, multiindex=False, random_data=True, random_ids=True) library.write(symbol, large_ts[0:1000]) #v1 library.snapshot('snap_a') library.append(symbol, large_ts[1000:1010]) #v2 library.snapshot('snap_b') library.append(symbol, large_ts[1010:1020]) #v3 library.snapshot('snap_c') library.append(symbol, large_ts[1030:1040]) #v4 library.delete(symbol) #v5 library.append(symbol, large_ts[1040:1050], upsert=False) # v6 last_v = library._versions.find_one(sort=[('version', pymongo.DESCENDING)]) vsu.analyze_symbol(library, symbol, 0, last_v['version'] + 1) # Verify no versions have been corrupted for v in library._versions.find(sort=[('version', pymongo.DESCENDING)]): library.read(symbol, as_of=v['version'])
def _corrupt_with_append_only(library, library_name): def do_fail(version): raise Exception('test') large_ts = create_test_data(size=2000, cols=100, index=True, multiindex=False, random_data=True, random_ids=True) library.write(symbol, large_ts[0:1000]) # v1 library.snapshot('snap_write_a') library.append(symbol, large_ts[1000:1010]) # v2 library.snapshot('snap_write_b') # Here we simulate a scenario where an append succeeds to insert the data segments, # but fails to insert the version document (i.e. Mongo error occurred) orig_insert_version = library._insert_version library._insert_version = do_fail try: library.append(symbol, large_ts[1010:1020]) # v3 except: pass library._insert_version = orig_insert_version library.write_metadata(symbol, {'hello': 'there'}) # , prune_previous_version=False) # Appending subsequently overlapping and non-SHA-matching data cause data corruption library.append(symbol, large_ts[1018:1030]) # , prune_previous_version=False) last_v = library._versions.find_one(sort=[('version', pymongo.DESCENDING)]) vsu.analyze_symbol(library, symbol, 0, last_v['version'] + 1) # Verify no versions have been corrupted for v in library._versions.find(sort=[('version', pymongo.DESCENDING)]): library.read(symbol, as_of=v['version'])
def test_no_corruption_restore_append_non_overlapping_tstamps(library, library_name): large_ts = create_test_data(size=2000, cols=100, index=True, multiindex=False, random_data=True, random_ids=True) # Append with 50 small uncompressed segments (no new base yet) last_row_b = n_append(library, library_name, 50, 25, large_ts, 0, False, True) library.snapshot('snap_A') # Append with 20 more small segments, causes once copy-rewrite with new base, and then some small appended segments n_append(library, library_name, 15, 25, large_ts, last_row_b, True, True) library.restore_version(symbol, as_of='snap_A') last_row = n_append(library, library_name, 1, 40, large_ts, last_row_b, False, True) library.snapshot('snap_B') # Corrupts all versions last_row = n_append(library, library_name, 1, 10, large_ts, last_row, False, True) last_row = n_append(library, library_name, 8, 20, large_ts, last_row, False, True) library.snapshot('snap_C') last_v = library._versions.find_one(sort=[('version', pymongo.DESCENDING)]) vsu.analyze_symbol(library, symbol, 0, last_v['version'] + 1) # Verify no versions have been corrupted for v in library._versions.find(sort=[('version', pymongo.DESCENDING)]): library.read(symbol, as_of=v['version'])
def test_restore_append_overlapping_corrupts_last(library, library_name): large_ts = create_test_data(size=2000, cols=100, index=True, multiindex=False, random_data=True, random_ids=True) library.write(symbol, large_ts[0:1000]) library.snapshot('snap_write_a') library.append(symbol, large_ts[1000:1010]) library.restore_version(symbol, as_of='snap_write_a', prune_previous_version=True) library.append(symbol, large_ts[1000:1012]) last_v = library._versions.find_one(sort=[('version', pymongo.DESCENDING)]) vsu.analyze_symbol(library, symbol, 0, last_v['version'] + 1) # Verify no versions have been corrupted for v in library._versions.find(sort=[('version', pymongo.DESCENDING)]): library.read(symbol, as_of=v['version'])
def _corrupt_with_append_only(library, library_name): def do_fail(version): raise Exception('test') large_ts = create_test_data(size=2000, cols=100, index=True, multiindex=False, random_data=True, random_ids=True) library.write(symbol, large_ts[0:1000]) # v1 library.snapshot('snap_write_a') library.append(symbol, large_ts[1000:1010]) # v2 library.snapshot('snap_write_b') # Here we simulate a scenario where an append succeeds to insert the data segments, # but fails to insert the version document (i.e. Mongo error occurred) orig_insert_version = library._insert_version library._insert_version = do_fail try: library.append(symbol, large_ts[1010:1020]) # v3 except: pass library._insert_version = orig_insert_version library.write_metadata( symbol, {'hello': 'there'}) # , prune_previous_version=False) # Appending subsequently overlapping and non-SHA-matching data cause data corruption library.append(symbol, large_ts[1018:1030]) # , prune_previous_version=False) last_v = library._versions.find_one(sort=[('version', pymongo.DESCENDING)]) vsu.analyze_symbol(library, symbol, 0, last_v['version'] + 1) # Verify no versions have been corrupted for v in library._versions.find(sort=[('version', pymongo.DESCENDING)]): library.read(symbol, as_of=v['version'])
def test_no_corruption_restore_append_non_overlapping_tstamps( library, library_name): large_ts = create_test_data(size=2000, cols=100, index=True, multiindex=False, random_data=True, random_ids=True) # Append with 50 small uncompressed segments (no new base yet) last_row_b = n_append(library, library_name, 50, 25, large_ts, 0, False, True) library.snapshot('snap_A') # Append with 20 more small segments, causes once copy-rewrite with new base, and then some small appended segments n_append(library, library_name, 15, 25, large_ts, last_row_b, True, True) library.restore_version(symbol, as_of='snap_A') last_row = n_append(library, library_name, 1, 40, large_ts, last_row_b, False, True) library.snapshot('snap_B') # Corrupts all versions last_row = n_append(library, library_name, 1, 10, large_ts, last_row, False, True) last_row = n_append(library, library_name, 8, 20, large_ts, last_row, False, True) library.snapshot('snap_C') last_v = library._versions.find_one(sort=[('version', pymongo.DESCENDING)]) vsu.analyze_symbol(library, symbol, 0, last_v['version'] + 1) # Verify no versions have been corrupted for v in library._versions.find(sort=[('version', pymongo.DESCENDING)]): library.read(symbol, as_of=v['version'])