def test_persist(tmpdir): dam = DocumentArrayMemmap(tmpdir) docs = list(random_docs(100)) for doc in docs: doc.scores['score'] = 50 doc.evaluations['eval'] = 100 dam.extend(docs) dam2 = DocumentArrayMemmap(tmpdir) assert len(dam2) == 100 assert dam == dam2 for d1, d2 in zip(dam, dam2): assert d1.proto == d2.proto assert '1' in dam del dam['1'] assert len(dam2) == 100 dam2.reload() assert len(dam2) == 99 for doc2 in dam2: assert doc2.scores['score'].value == 50 assert doc2.evaluations['eval'].value == 100 dam.clear() assert len(dam2) == 99 dam2.reload() assert len(dam2) == 0
def memmap_with_text_and_embedding(tmpdir): dam = DocumentArrayMemmap(tmpdir) for idx in range(100): d = Document(text=f'random text {idx}', embedding=np.random.rand(512)) dam.append(d) yield dam dam.clear()
def test_error(tmpdir): dam = DocumentArrayMemmap(tmpdir) dam.clear() with pytest.raises(KeyError): dam['12'] with pytest.raises(IndexError): dam[1] with pytest.raises(IndexError): del dam[1] with pytest.raises(KeyError): del dam['12']
def test_convert_dam_to_da(tmpdir, mocker): dam = DocumentArrayMemmap(tmpdir) dam.extend(random_docs(100)) da = DocumentArray(dam) dam.clear() mock = mocker.Mock() for d in da: assert d mock() mock.assert_called() assert len(da) == 100 assert len(dam) == 0
def test_memmap_mutate(tmpdir): da = DocumentArrayMemmap(tmpdir) d0 = Document(text='hello') da.append(d0) assert da[0] == d0 d1 = Document(text='world') da.append(d1) assert da[1] == d1 da2 = DocumentArrayMemmap(tmpdir) assert len(da2) == 2 assert da2[0] == d0 assert da2[1] == d1 da.clear() assert not len(da)
def test_extend_and_get_attribute(tmpdir, embed_dim): dam = DocumentArrayMemmap(tmpdir) dam.clear() docs = list(random_docs(100, start_id=0, embed_dim=embed_dim)) dam.extend(docs) dam2 = DocumentArrayMemmap(tmpdir) x = dam2.get_attributes('embedding') assert len(dam2) == 100 docs = list(random_docs(100, start_id=100, embed_dim=embed_dim)) dam2.extend(docs) x = dam2.get_attributes('embedding') assert len(x) == 200 assert x[0].shape == (embed_dim, ) assert len(dam2) == 200
def test_memmap_delete_clear(tmpdir, mocker, idx1, idx99): dam = DocumentArrayMemmap(tmpdir) candidates = list(random_docs(100)) for d in candidates: d.id = f'id_{d.id}' dam.extend(candidates) assert len(dam) == 100 del dam[idx1] assert len(dam) == 99 del dam[idx99 if isinstance(idx99, str) else idx99 - 1] assert len(dam) == 98 for d in dam: assert d.id != idx1 assert d.id != idx99 dam.clear() assert len(dam) == 0 mock = mocker.Mock() for _ in dam: mock() mock.assert_not_called()