def test_append_only_array_storage_iter():
with tempfile.NamedTemporaryFile() as tmp_file:
storage = AppendOnlyArrayStorage(tmp_file.name)
for i in range(100):
storage.append(i + 10)
for index, value in enumerate(storage):
assert value == index + 10
def test_append_only_array_storage_append():
with tempfile.NamedTemporaryFile() as tmp_file:
storage = AppendOnlyArrayStorage(tmp_file.name)
for i in range(10):
storage.append(i)
storage.append("hehe")
storage.append({"a": "b"})
storage.append({"a", "b", "c"})
assert len(storage) == 13
class IndexReader(object):
def __init__(self, storage_filepath):
self._storage = AppendOnlyArrayStorage(storage_filepath)
def __iter__(self):
for dict_item in self._storage:
if isinstance(dict_item, dict):
yield dict_item["features"], dict_item["line_number"], dict_item["next_line"]
def close(self):
self._storage.close()
def test_append_only_array_storage_get():
with tempfile.NamedTemporaryFile() as tmp_file:
storage = AppendOnlyArrayStorage(tmp_file.name)
for i in range(10):
storage.append(i)
storage.append("hehe")
storage.append({"a": "b"})
storage.append({"a", "b", "c"})
for i in range(10):
assert storage[i] == i
assert storage[10] == "hehe"
assert storage[11] == {"a": "b"}
assert storage[12] == {"a", "b", "c"}
class IndexWriter(object):
def __init__(self, storage_filepath):
self._storage = AppendOnlyArrayStorage(storage_filepath)
def write(self, features, line_number, next_line):
self._storage.append({
"features": features,
"line_number": line_number,
"next_line": next_line
})
def close(self):
self._storage.close()
class CorpusWriter(object):
def __init__(self, storage_filepath, flush=False):
self._storage = AppendOnlyArrayStorage(storage_filepath, flush)
def write_corpus(self, code, features, filepath):
"""
very straightforward solution now...
"""
self._storage.append({
"code": code,
"features": features,
"filepath": filepath
})
def close(self):
self._storage.close()
class CorpusReader(object):
def __init__(self, storage_filepath):
self._storage = AppendOnlyArrayStorage(storage_filepath)
def __iter__(self):
for dict_item in self._storage:
# since we use dict as item storage, double check item type here
if isinstance(dict_item, dict):
yield dict_item["code"], dict_item["features"], dict_item[
"filepath"]
def __getitem__(self, item):
return self._storage[item]
def close(self):
self._storage.close()
def __init__(self, storage_filepath):
self._storage = AppendOnlyArrayStorage(storage_filepath)
def __init__(self, storage_filepath, flush=False):
self._storage = AppendOnlyArrayStorage(storage_filepath, flush)