def test_multiprocessing(self):
num_frames = 10
content = get_test_indexed_dataset_content(type_name="float64",
num_frames=num_frames,
single_frame_dims=(128,
128))
with ExitStack() as stack:
stack.enter_context(
patch(
"frldistml.scaffold.storage_layers.posix_storage.np.memmap",
side_effect=mock_np_content(content),
))
stack.enter_context(
patch(
"frldistml.scaffold.storage_layers.posix_storage.np.fromfile",
side_effect=mock_np_content(content),
))
dataset = PosixIndexedDatasetReader(
idxfile=StoragePath(FILE_BASE + ".idx"),
binfile=StoragePath(FILE_BASE + ".bin"),
)
frames = {idx: dataset[idx] for idx in [0, len(dataset) - 2]}
p = multiprocessing.Process(target=mp_dataset_worker,
args=(dataset, len(dataset), frames))
p.start()
p.join()
self.assertEqual(p.exitcode, 0)
def test_write_read(self):
data1 = np.arange(100).reshape(10, 5, 2)
data2 = data1 + 1000
with TemporaryDirectory() as d:
idxpath = StoragePath(os.path.join(d, "dataset.idx"))
binpath = StoragePath(os.path.join(d, "dataset.bin"))
with IndexedDatasetWriterFactory.get(idxfile=idxpath,
binfile=binpath) as writer:
writer.push_back(data1)
writer.push_back(data2)
reader = PosixIndexedDatasetReader(idxfile=idxpath,
binfile=binpath)
self.assertTrue(np.array_equal(data1, reader[0]))
self.assertTrue(np.array_equal(data2, reader[1]))
def test_reading_element(self):
num_frames = 10
single_frame_dims = (128, 128)
content = get_test_indexed_dataset_content(
type_name="float64",
num_frames=num_frames,
single_frame_dims=single_frame_dims,
)
with ExitStack() as stack:
stack.enter_context(
patch(
"frldistml.scaffold.storage_layers.posix_storage.np.memmap",
side_effect=mock_np_content(content),
))
stack.enter_context(
patch(
"frldistml.scaffold.storage_layers.posix_storage.np.fromfile",
side_effect=mock_np_content(content),
))
dataset = MultifieldIndexedDataset(folder_path=StoragePath(""),
fields=[FIELD],
filenames=[FILE_BASE])
self.assertEqual(len(dataset), num_frames)
self.assertEqual(dataset[0][FIELD].shape, single_frame_dims)
self.assertEqual(dataset[num_frames - 1][FIELD].shape,
single_frame_dims)
self.assertTrue(check_frame_equality(dataset, 0,
single_frame_dims))
def test_storage_os_path_exists_posix_explicit(self):
with patch(
"frldistml.scaffold.storage_layers.posix_storage.PosixStorageLayer.exists"
) as os_path_exists:
storage_os.path.exists(StoragePath(self.posix_explicit_path))
os_path_exists.assert_called_once()
def test_storage_os_path_getsize_posix_implicit(self):
with patch(
"frldistml.scaffold.storage_layers.posix_storage.PosixStorageLayer.getsize"
) as os_path_getsize:
storage_os.path.getsize(StoragePath(self.posix_implicit_path))
os_path_getsize.assert_called_once()
def test_storage_os_makedirs_posix_implicit(self):
with patch(
"frldistml.scaffold.storage_layers.posix_storage.PosixStorageLayer.makedirs"
) as os_makedirs:
storage_os.makedirs(StoragePath(self.posix_implicit_path))
os_makedirs.assert_called_once()
def test_storage_open_posix_explicit(self):
with patch("builtins.open", mock_open()) as open_mock:
with storage_open(StoragePath(self.posix_explicit_path), "rb"):
open_mock.assert_called_once()
def test_invalid_explicit(self):
storage_path = StoragePath(self.invalid_explicit_path)
with self.assertRaises(ValueError):
StorageLayerFactory.get(storage_path)
def test_posix_implicit(self):
str_path = "test"
storage_path = StoragePath(str_path)
self.assertEqual(storage_path.scheme, PosixStorageType.scheme)
self.assertEqual(storage_path.path, PurePath(str_path))
def test_posix_serialize_deserialize(self):
orig = StoragePath(self.posix_explicit_path)
recreated = StoragePath(str(orig))
self.assertEqual(orig.path, recreated.path)