def test_persist(tmpdir):
dam = DocumentArrayMemmap(tmpdir)
docs = list(random_docs(100))
for doc in docs:
doc.scores['score'] = 50
doc.evaluations['eval'] = 100
dam.extend(docs)
dam2 = DocumentArrayMemmap(tmpdir)
assert len(dam2) == 100
assert dam == dam2
for d1, d2 in zip(dam, dam2):
assert d1.proto == d2.proto
assert '1' in dam
del dam['1']
assert len(dam2) == 100
dam2.reload()
assert len(dam2) == 99
for doc2 in dam2:
assert doc2.scores['score'].value == 50
assert doc2.evaluations['eval'].value == 100
dam.clear()
assert len(dam2) == 99
dam2.reload()
assert len(dam2) == 0
def memmap_with_text_and_embedding(tmpdir):
dam = DocumentArrayMemmap(tmpdir)
for idx in range(100):
d = Document(text=f'random text {idx}', embedding=np.random.rand(512))
dam.append(d)
yield dam
dam.clear()
def test_error(tmpdir):
dam = DocumentArrayMemmap(tmpdir)
dam.clear()
with pytest.raises(KeyError):
dam['12']
with pytest.raises(IndexError):
dam[1]
with pytest.raises(IndexError):
del dam[1]
with pytest.raises(KeyError):
del dam['12']
def test_convert_dam_to_da(tmpdir, mocker):
dam = DocumentArrayMemmap(tmpdir)
dam.extend(random_docs(100))
da = DocumentArray(dam)
dam.clear()
mock = mocker.Mock()
for d in da:
assert d
mock()
mock.assert_called()
assert len(da) == 100
assert len(dam) == 0
def test_memmap_mutate(tmpdir):
da = DocumentArrayMemmap(tmpdir)
d0 = Document(text='hello')
da.append(d0)
assert da[0] == d0
d1 = Document(text='world')
da.append(d1)
assert da[1] == d1
da2 = DocumentArrayMemmap(tmpdir)
assert len(da2) == 2
assert da2[0] == d0
assert da2[1] == d1
da.clear()
assert not len(da)
def test_extend_and_get_attribute(tmpdir, embed_dim):
dam = DocumentArrayMemmap(tmpdir)
dam.clear()
docs = list(random_docs(100, start_id=0, embed_dim=embed_dim))
dam.extend(docs)
dam2 = DocumentArrayMemmap(tmpdir)
x = dam2.get_attributes('embedding')
assert len(dam2) == 100
docs = list(random_docs(100, start_id=100, embed_dim=embed_dim))
dam2.extend(docs)
x = dam2.get_attributes('embedding')
assert len(x) == 200
assert x[0].shape == (embed_dim, )
assert len(dam2) == 200
def test_memmap_delete_clear(tmpdir, mocker, idx1, idx99):
dam = DocumentArrayMemmap(tmpdir)
candidates = list(random_docs(100))
for d in candidates:
d.id = f'id_{d.id}'
dam.extend(candidates)
assert len(dam) == 100
del dam[idx1]
assert len(dam) == 99
del dam[idx99 if isinstance(idx99, str) else idx99 - 1]
assert len(dam) == 98
for d in dam:
assert d.id != idx1
assert d.id != idx99
dam.clear()
assert len(dam) == 0
mock = mocker.Mock()
for _ in dam:
mock()
mock.assert_not_called()
