def test_read(self):
name = "my_name"
train_info = SplitInfo(name="train", num_examples=100)
test_info = SplitInfo(name="test", num_examples=100)
split_infos = [train_info, test_info]
split_dict = SplitDict()
split_dict.add(train_info)
split_dict.add(test_info)
info = DatasetInfo(splits=split_dict)
reader = ReaderTest("", info)
instructions = "test[:33%]"
dset = Dataset(**reader.read(name, instructions, split_infos))
self.assertEqual(dset["filename"][0], f"{name}-test")
self.assertEqual(dset.num_rows, 33)
self.assertEqual(dset.num_columns, 1)
instructions = ["train", "test[:33%]"]
datasets_kwargs = [
reader.read(name, instr, split_infos) for instr in instructions
]
train_dset, test_dset = [
Dataset(**dataset_kwargs) for dataset_kwargs in datasets_kwargs
]
self.assertEqual(train_dset["filename"][0], f"{name}-train")
self.assertEqual(train_dset.num_rows, 100)
self.assertEqual(train_dset.num_columns, 1)
self.assertEqual(test_dset["filename"][0], f"{name}-test")
self.assertEqual(test_dset.num_rows, 33)
self.assertEqual(test_dset.num_columns, 1)
def finalize(self, timeout=100):
"""Close all the writing process and load/gather the data
from all the nodes if main node or all_process is True.
"""
if self.writer is not None:
self.writer.finalize()
self.writer = None
if self.filelock is not None:
self.filelock.release()
if self.keep_in_memory:
# Read the predictions and references
reader = ArrowReader(path=self.data_dir, info=None)
self.data = Dataset.from_buffer(self.buf_writer.getvalue())
elif self.process_id == 0:
# Let's acquire a lock on each node files to be sure they are finished writing
file_paths, filelocks = self._get_all_cache_files(timeout=timeout)
# Read the predictions and references
try:
reader = ArrowReader(path=self.data_dir, info=None)
self.data = Dataset(**reader.read_files([{
"filename": f
} for f in file_paths]))
except FileNotFoundError:
raise ValueError(
"Another metric instance is already using the local cache file. "
"Please specify an experiment_id to avoid colision between distributed metric instances."
)
# Store file paths and locks and we will release/delete them after the computation.
self.file_paths = file_paths
self.filelocks = filelocks
def _create_dummy_dataset(self) -> Dataset:
dset = Dataset(
pa.Table.from_pydict({
"filename": [
"my_name-train" + "_" + str(x)
for x in np.arange(30).tolist()
]
}))
return dset
def _create_dummy_dataset(self):
name = "my_name"
train_info = SplitInfo(name="train", num_examples=30)
test_info = SplitInfo(name="test", num_examples=30)
split_infos = [train_info, test_info]
split_dict = SplitDict()
split_dict.add(train_info)
split_dict.add(test_info)
info = DatasetInfo(splits=split_dict)
reader = ReaderTester("", info)
dset = Dataset(**reader.read(name, "train", split_infos))
return dset
def test_read_files(self):
train_info = SplitInfo(name="train", num_examples=100)
test_info = SplitInfo(name="test", num_examples=100)
split_dict = SplitDict()
split_dict.add(train_info)
split_dict.add(test_info)
info = DatasetInfo(splits=split_dict)
reader = ReaderTest("", info)
files = [{
"filename": "train"
}, {
"filename": "test",
"skip": 10,
"take": 10
}]
dset = Dataset(**reader.read_files(files, original_instructions=""))
self.assertEqual(dset.num_rows, 110)
self.assertEqual(dset.num_columns, 1)
self.assertEqual(dset._data_files, files)