def test_write_append(monkeypatch):
patch(monkeypatch)
fragment = FragmentedJsonList(data_dir, fragment_size=fragment_size)
fragment.append({"data": "thunder"})
assert fragment[-1]["data"] == "thunder"
assert len(fragment.dirty_fragments) == 1
def test_write_extend(monkeypatch):
patch(monkeypatch)
fragment = FragmentedJsonList(data_dir, fragment_size=fragment_size)
extension = [1, 2, 3]
fragment.extend(extension)
assert fragment[-len(extension):] == extension
assert len(fragment.dirty_fragments) >= 1
def test_write_extend_large(monkeypatch, mocker):
patch(monkeypatch)
new_fragment_spy = mocker.spy(FragmentedJsonList, "create_new_fragment")
fragment = FragmentedJsonList(data_dir, fragment_size=fragment_size)
fragments_to_create = 20
extension = list(range(fragment_size * fragments_to_create))
fragment.extend(extension)
assert fragment[-len(extension):] == extension
assert new_fragment_spy.call_count == fragments_to_create
def test_write_update(monkeypatch):
patch(monkeypatch)
fragment = FragmentedJsonList(data_dir, fragment_size=fragment_size)
update_len = fragment_size
for i in range(update_len):
fragment.update(i, {
"something": i
})
for i in range(update_len):
assert "something" in fragment[i] and fragment[i]["something"] == i
assert len(fragment.dirty_fragments) == 1 and 0 in fragment.dirty_fragments
def test_read_basic(monkeypatch):
patch(monkeypatch)
fragment = FragmentedJsonList(data_dir, fragment_size=fragment_size)
assert len(fragment) == fragment_size * (num_fragments - 1) + last_fragment_size
counter = 0
for i in fragment:
assert "data" in i
assert i["data"] == counter
counter += 1
assert fragment[0]["data"] == 0
assert fragment[len(fragment) - 1]["data"] == len(fragment) - 1
def test_write_commit(monkeypatch, mocker):
patch(monkeypatch)
write_json_spy = mocker.spy(FragmentedJsonList, "_write_json")
fragment = FragmentedJsonList(data_dir, fragment_size=fragment_size)
fragments_to_create = 20
extension = list(range(fragment_size * fragments_to_create + 1))
fragment.extend(extension)
assert len(fragment.dirty_fragments) in [20, 21]
fragment.commit_fragments()
assert len(fragment.dirty_fragments) == 0
assert write_json_spy.call_count in [20, 21]
def test_read_slice_reverse(monkeypatch):
patch(monkeypatch)
fragment = FragmentedJsonList(data_dir, fragment_size=fragment_size)
assert_slice(fragment, None, None, -1)
assert_slice(fragment, -5, -20, -2)
def test_read_slice_stepped(monkeypatch):
patch(monkeypatch)
fragment = FragmentedJsonList(data_dir, fragment_size=fragment_size)
assert_slice(fragment, 1, 21, 2)
def test_read_slice_with_nones(monkeypatch):
patch(monkeypatch)
fragment = FragmentedJsonList(data_dir, fragment_size=fragment_size)
assert_slice(fragment, None, 15, None)
assert_slice(fragment, 5, None, None)
def test_read_negative(monkeypatch):
patch(monkeypatch)
fragment = FragmentedJsonList(data_dir, fragment_size=fragment_size)
assert fragment[-1]["data"] == len(fragment) - 1
assert fragment[-1 * len(fragment)]["data"] == 0
from exporter.fragment import FragmentedJsonList
import os
import shutil
import sys
import tempfile
src_dir = sys.argv[1]
with tempfile.TemporaryDirectory() as tempdir:
src = FragmentedJsonList(src_dir)
dst = FragmentedJsonList(tempdir)
dst.extend(src[::-1])
src.close()
shutil.rmtree(src_dir)
src = FragmentedJsonList(src_dir)
src.extend(dst[::])
src.close()
dst.close()
print("OK")