def test_image_gray_float_csv_writeall(self):
# Read and write
dataset1 = spn.ImageDataset(
image_files=self.data_path("img_dir1/*-{*}.png"),
format=spn.ImageFormat.FLOAT,
num_epochs=1,
batch_size=2,
shuffle=False,
ratio=1,
crop=0,
accurate=True,
allow_smaller_final_batch=True)
writer = spn.CSVDataWriter(path=self.out_path(self.cid() + ".csv"))
data1 = dataset1.read_all()
dataset1.write_all(writer)
# Re-read
dataset2 = spn.CSVFileDataset(files=self.out_path(self.cid() + ".csv"),
num_vals=[None] * 25,
defaults=[[b'']] + [[1.0]
for _ in range(25)],
num_epochs=1,
batch_size=2,
shuffle=False,
num_labels=1,
allow_smaller_final_batch=True)
data2 = dataset2.read_all()
# Compare
np.testing.assert_allclose(data1[0], data2[0])
np.testing.assert_array_equal(data1[1], data2[1])
def test_csv_csv_writeall_tensorlist(self):
# Read&write
dataset1 = spn.CSVFileDataset(self.data_path("data_int1.csv"),
num_vals=[None] * 3,
defaults=[[101], [102], [103.0], [104.0],
[105.0]],
num_epochs=2,
batch_size=4,
shuffle=False,
num_labels=2,
min_after_dequeue=1000,
num_threads=1,
allow_smaller_final_batch=True)
path = self.out_path(self.cid() + ".csv")
writer = spn.CSVDataWriter(path)
data1 = dataset1.read_all()
dataset1.write_all(writer)
# Read again
dataset2 = spn.CSVFileDataset(path,
num_vals=[None] * 3,
defaults=[[201], [202], [203.0], [204.0],
[205.0]],
num_epochs=1,
batch_size=4,
shuffle=False,
num_labels=2,
min_after_dequeue=1000,
num_threads=1,
allow_smaller_final_batch=True)
data2 = dataset2.read_all()
# Compare
np.testing.assert_array_almost_equal(data1[0], data2[0])
np.testing.assert_array_equal(data1[1], data2[1])
def test_csv_data_writer(self):
# Write
path = self.out_path(self.cid() + ".csv")
writer = spn.CSVDataWriter(path)
arr1 = np.array([1, 2, 3, 4])
arr2 = np.array([[1 / 1, 1 / 2], [1 / 3, 1 / 4], [1 / 5, 1 / 6],
[1 / 7, 1 / 8]])
writer.write(arr2, arr1)
writer.write(arr2, arr1)
# Read
dataset = spn.CSVFileDataset(path,
num_vals=[None] * 2,
defaults=[[1], [1.0], [1.0]],
num_epochs=1,
batch_size=10,
shuffle=False,
num_labels=1,
min_after_dequeue=1000,
num_threads=1,
allow_smaller_final_batch=True)
data = dataset.read_all()
# Compare
np.testing.assert_array_almost_equal(np.concatenate((arr2, arr2)),
data[0])
np.testing.assert_array_equal(np.concatenate((arr1, arr1)),
data[1].flatten())