def test_read(self):
with Collection.open(self._fn_collection, mode='r') as c:
self.assertEqual(len(list(c)), 3)
with open(self._fn_collection, 'a') as file:
file.write("{'a': 0}\n") # ill-formed JSON (single quotes instead of double quotes)
with self.assertRaises(JSONParseError):
with Collection.open(self._fn_collection, mode='r') as c:
pass
def test_reopen(self):
docs = [dict(_id=str(i)) for i in range(10)]
with Collection.open(self._fn_collection) as c:
c.update(docs)
with Collection.open(self._fn_collection) as c:
self.assertEqual(len(c), len(docs))
for doc in self.c:
self.assertTrue(doc['_id'] in c)
def test_read(self):
with Collection.open(self._fn_collection, mode="r") as c:
assert len(list(c)) == 3
with open(self._fn_collection, "a") as file:
file.write(
"{'a': 0}\n"
) # ill-formed JSON (single quotes instead of double quotes)
with pytest.raises(JSONParseError):
with Collection.open(self._fn_collection, mode="r") as c:
pass
def test_copy(self):
docs = [dict(_id=str(i)) for i in range(10)]
self.c.update(docs)
c2 = Collection(self.c)
assert len(self.c) == len(c2)
for doc in c2:
assert len(self.c.find(doc)) == 1
def test_write_and_read(self):
self.c.to_json(self._fn_json)
assert os.path.getsize(self._fn_json) > 0
c = Collection.read_json(self._fn_json)
assert len(list(c)) == 10
assert len(c.find()) == 10
c.close()
def test_write_and_read(self):
self.c.to_json(self._fn_json)
self.assertGreater(os.path.getsize(self._fn_json), 0)
c = Collection.read_json(self._fn_json)
self.assertEqual(len(list(c)), 10)
self.assertEqual(len(c.find()), 10)
c.close()
def test_copy(self):
docs = [dict(_id=str(i)) for i in range(10)]
self.c.update(docs)
c2 = Collection(self.c)
self.assertEqual(len(self.c), len(c2))
for doc in c2:
self.assertEqual(len(self.c.find(doc)), 1)
def test_init_with_list_with_and_without_ids(self):
docs = [{'a': i} for i in range(10)]
for i, doc in enumerate(islice(docs, 5)):
doc.setdefault('_id', str(i))
self.c = Collection(docs)
self.assertEqual(len(self.c), len(docs))
for doc in docs:
self.assertIn(doc['_id'], self.c)
def test_file_size(self):
docs = [dict(_id=str(i)) for i in range(10)]
with open(self._fn_collection) as f:
assert len(list(f)) == 0
with Collection.open(self._fn_collection) as c:
c.update(docs)
with open(self._fn_collection) as f:
assert len(list(f)) == len(docs)
with Collection.open(self._fn_collection) as c:
assert len(c) == len(docs)
for doc in docs:
c.replace_one({"_id": doc["_id"]}, doc)
with Collection.open(self._fn_collection) as c:
assert len(c) == len(docs)
with open(self._fn_collection) as f:
assert len(list(f)) == len(docs)
def setUp(self, request):
self._tmp_dir = TemporaryDirectory(prefix="signac_collection_")
request.addfinalizer(self._tmp_dir.cleanup)
self._fn_json = os.path.join(self._tmp_dir.name, "test.json")
self.c = Collection.open(filename=":memory:")
docs = [dict(_id=str(i)) for i in range(10)]
self.c.update(docs)
self.c.flush()
def setUp(self):
self._tmp_dir = TemporaryDirectory(prefix='signac_collection_')
self._fn_json = os.path.join(self._tmp_dir.name, 'test.json')
self.addCleanup(self._tmp_dir.cleanup)
self.c = Collection.open(filename=':memory:')
docs = [dict(_id=str(i)) for i in range(10)]
self.c.update(docs)
self.c.flush()
def test_file_size(self):
docs = [dict(_id=str(i)) for i in range(10)]
with open(self._fn_collection) as f:
self.assertEqual(len(list(f)), 0)
with Collection.open(self._fn_collection) as c:
c.update(docs)
with open(self._fn_collection) as f:
self.assertEqual(len(list(f)), len(docs))
with Collection.open(self._fn_collection) as c:
self.assertEqual(len(c), len(docs))
for doc in docs:
c.replace_one({'_id': doc['_id']}, doc)
with Collection.open(self._fn_collection) as c:
self.assertEqual(len(c), len(docs))
with open(self._fn_collection) as f:
self.assertEqual(len(list(f)), len(docs))
def test_init_with_list_with_and_without_ids(self):
docs = [{"a": i} for i in range(10)]
for i, doc in enumerate(islice(docs, 5)):
doc.setdefault("_id", str(i))
self.c = Collection(docs)
assert len(self.c) == len(docs)
for doc in docs:
assert doc["_id"] in self.c
def test_write_flush_and_reopen(self):
docs = [dict(_id=str(i)) for i in range(10)]
self.c.update(docs)
self.c.flush()
self.assertGreater(os.path.getsize(self._fn_collection), 0)
with Collection.open(self._fn_collection) as c:
self.assertEqual(len(c), len(docs))
for doc in self.c:
self.assertIn(doc['_id'], c)
def test_write_flush_and_reopen(self):
docs = [dict(_id=str(i)) for i in range(10)]
self.c.update(docs)
self.c.flush()
assert os.path.getsize(self._fn_collection) > 0
with Collection.open(self._fn_collection) as c:
assert len(c) == len(docs)
for doc in self.c:
assert doc["_id"] in c
def test_write_on_readonly(self):
c = Collection.open(self._fn_collection, mode='r')
self.assertEqual(len(list(c)), 10)
c.insert_one(dict())
self.assertEqual(len(list(c)), 11)
with self.assertRaises(io.UnsupportedOperation):
c.flush()
with self.assertRaises(io.UnsupportedOperation):
c.close()
with self.assertRaises(RuntimeError):
c.find()
def test_write_on_readonly(self):
c = Collection.open(self._fn_collection, mode="r")
assert len(list(c)) == 10
c.insert_one(dict())
assert len(list(c)) == 11
with pytest.raises(io.UnsupportedOperation):
c.flush()
with pytest.raises(io.UnsupportedOperation):
c.close()
with pytest.raises(RuntimeError):
c.find()
def test_compression_level(self):
docs = [dict(_id=str(i)) for i in range(10)]
self.c.update(docs)
self.c.flush()
fn_txt = self._fn_collection + ".txt"
with Collection.open(fn_txt) as c_text:
c_text.update(self.c)
size_txt = os.path.getsize(fn_txt)
size_gz = os.path.getsize(self._fn_collection)
assert size_txt > 0
assert size_gz > 0
compresslevel = size_txt / size_gz
assert compresslevel > 1.0
def test_compression_level(self):
docs = [dict(_id=str(i)) for i in range(10)]
self.c.update(docs)
self.c.flush()
fn_txt = self._fn_collection + '.txt'
with Collection.open(fn_txt) as c_text:
c_text.update(self.c)
size_txt = os.path.getsize(fn_txt)
size_gz = os.path.getsize(self._fn_collection)
self.assertGreater(size_txt, 0)
self.assertGreater(size_gz, 0)
compresslevel = size_txt / size_gz
self.assertGreater(compresslevel, 1.0)
def test_compression(self):
# Fill with data
docs = [dict(_id=str(i)) for i in range(10)]
self.c.update(docs)
self.c.flush()
# Create uncompressed copy to compare to.
c2 = Collection(self.c)
c2.flush()
self.assertIsInstance(self.c._file, io.BytesIO)
size_compressed = len(self.c._file.getvalue())
self.assertGreater(size_compressed, 0)
size_uncompressed = len(c2._file.getvalue().encode('utf-8'))
self.assertGreater(size_uncompressed, 0)
compresslevel = size_uncompressed / size_compressed
self.assertGreater(compresslevel, 1.0)
def test_compression(self):
# Fill with data
docs = [dict(_id=str(i)) for i in range(10)]
self.c.update(docs)
self.c.flush()
# Create uncompressed copy to compare to.
c2 = Collection(self.c)
c2.flush()
assert isinstance(self.c._file, io.BytesIO)
size_compressed = len(self.c._file.getvalue())
assert size_compressed > 0
size_uncompressed = len(c2._file.getvalue().encode("utf-8"))
assert size_uncompressed > 0
compresslevel = size_uncompressed / size_compressed
assert compresslevel > 1.0
def test_init_with_list_with_ids_sequential(self):
docs = [{"a": i, "_id": str(i)} for i in range(10)]
self.c = Collection(docs)
assert len(self.c) == len(docs)
for doc in docs:
assert doc["_id"] in self.c
def setUp(self, request):
self._tmp_dir = TemporaryDirectory(prefix="signac_collection_")
request.addfinalizer(self._tmp_dir.cleanup)
self._fn_collection = os.path.join(self._tmp_dir.name, self.filename)
self.c = Collection.open(self._fn_collection, mode=self.mode)
request.addfinalizer(self.c.close)
def test_init_with_list_with_ids_non_sequential(self):
docs = [{"a": i, "_id": "{:032d}".format(i**3)} for i in range(10)]
self.c = Collection(docs)
assert len(self.c) == len(docs)
for doc in docs:
assert doc["_id"] in self.c
def test_read(self):
c = Collection.open(self._fn_collection, mode="r")
assert len(list(c)) == 10
assert len(list(c)) == 10
assert len(c.find()) == 10
c.close()
def setUp(self, request):
self._tmp_dir = TemporaryDirectory(prefix="signac_collection_")
request.addfinalizer(self._tmp_dir.cleanup)
self._fn_collection = os.path.join(self._tmp_dir.name, "test.txt")
with Collection.open(self._fn_collection, "w") as c:
c.update([dict(_id=str(i)) for i in range(10)])
def test_init_with_list_without_ids(self):
docs = [{"a": i} for i in range(10)]
self.c = Collection(docs)
assert len(self.c) == len(docs)
for doc in docs:
assert doc["_id"] in self.c
def setUp(self):
self.c = Collection.open(filename=":memory:", compresslevel=9)
def get_index_collection(self):
c = Collection()
return Mock(spec=c, wraps=c)
def test_init_with_non_serializable(self):
docs = [dict(a=array.array("f", [1, 2, 3])) for i in range(10)]
with pytest.raises(TypeError):
self.c = Collection(docs)
