def test_append_to_existing(tmp_path):
""" Test that we can add new records to existing file """
# Write file
pfile = tmp_path / "test.flt"
with pfile.open('wb') as fhandle, FileWriter(fhandle) as writer:
a = writer.append(100, [0])
b = writer.append(101, [1])
c = writer.append(102, [2])
# Insert new record to existing file
with pfile.open('rb+') as fhandle, FileWriter(fhandle) as writer:
d = writer.append(103, [3])
# Read and validate
with pfile.open('rb') as fhandle, FileReader(fhandle) as reader:
assert set(reader.keys()) == {100, 101, 102, 103}
assert reader.read(103) == [3]
def test_bench_simple(key_count, benchmark):
memfile = io.BytesIO()
with FileWriter(memfile) as writer:
for key in range(key_count):
writer.append(str(key), [int(key)])
res = benchmark(read_in_order, memfile)
assert res == key_count
def test_append_to_new(tmp_path):
"""Test that we can append to an empty file"""
# Use append to write initial records into a new file (opened in 'wb' mode)
pfile = tmp_path / "test.flt"
with pfile.open('wb') as fhandle, FileWriter(fhandle) as writer:
a = writer.append(100, [0])
b = writer.append(101, [1])
# Read and validate
with pfile.open('rb') as fhandle, FileReader(fhandle) as reader:
assert set(reader.keys()) == {100, 101}
assert reader.read(101) == [1]
def test_read_twice(tmp_path):
# Write file
pfile = tmp_path / "test.flt"
with pfile.open('wb') as fhandle, FileWriter(fhandle) as writer:
writer.append(100, [1, 2, 3])
with pfile.open('rb') as fhandle:
with FileReader(fhandle) as reader:
assert reader.keys() == {100}
with FileReader(fhandle) as reader:
assert reader.keys() == {100}
def test_basic(tmp_path):
""" Test that we can read the entirety of a simple file"""
# Write file
pfile = tmp_path / "test.flt"
with pfile.open('wb') as fhandle, FileWriter(fhandle) as writer:
writer.append(100, [1, 2, 3])
# Read and validate
with pfile.open('rb') as fhandle, FileReader(fhandle) as reader:
assert set(reader.keys()) == {100}
rec = reader.read(100)
assert len(rec) == 3
assert rec == [1, 2, 3]
def test_read_complex(tmp_path):
""" Test that we can read a set of keys """
# Write file
pfile = tmp_path / "test.flt"
with pfile.open('wb') as fhandle, FileWriter(fhandle) as writer:
writer.append(100, [1, 2, 3])
writer.append(101, [4, 5, 6])
writer.append("alpha", ["7", "8", "9"])
# Read and validate
with pfile.open('rb') as fhandle, FileReader(fhandle) as reader:
assert set(reader.keys()) == {100, 101, "alpha"}
rec = reader.read([100, 101, "alpha"])
assert np.array(rec).shape == (3, 3)
assert rec[0] == [1, 2, 3]
assert rec[1] == [4, 5, 6]
assert rec[2] == ["7", "8", "9"]
# In this example, each patient record includes results from 15 ECG leads.
# Leads #1-12 each contain 1248 values, and leads #13-15 each contain 5040 values.
#
# Rather than save these 15 results sets separately, we save each patient record
# as a single list of 15 arrays, which simplifies data management and improves data
# load times during training.
from fleetfmt import FileWriter, FileReader
import numpy as np
from pathlib import Path
fleetfile = Path("./random_example.flt")
# Number of patient records
numkeys = 10000
# Utility to create a test numpy array
def make_dataset():
def _rand_vec(size):
data = np.array(1000 * np.random.random(size), dtype=np.float32)
return data
return [_rand_vec(1248) for _ in range(12)] + \
[_rand_vec(5040) for _ in range(3)]
with fleetfile.open('wb') as fhandle, FileWriter(fhandle) as writer:
for key in range(0, numkeys):
writer.append(key, [*make_dataset()])