def test_read_to_fit_memory_dangling_element(tmpdir_factory, output_dtype):
"""Test that data is read in correctly when `len(data) = 1 mod batch_size`."""
data = np.zeros((10, 10)).astype(str)
for i in range(data.shape[0]):
data[i, i] = str(i + 1)
data_dir = tmpdir_factory.mktemp("ten_line_csv")
data_file = data_dir.join("ten_lines.csv")
np.savetxt(data_file.strpath, data, delimiter=",", newline="\n", fmt="%s")
X_read, y_read = _read_to_fit_memory(
_get_reader(data_dir.strpath, 3),
psutil.virtual_memory().total,
output_dtype=output_dtype,
target_column_index=0,
)
assert np.array_equal(data[:, 1:], X_read)
assert np.array_equal(data[:, 0], y_read)
def test_get_reader_incorrect_path():
"""Test for reading from a path that doesn't exist"""
with pytest.raises(RuntimeError):
_get_reader(source="incorrect", batch_size=100)
def test_get_reader_error_malformed_channel_cfg(cfg, expected_error):
"""Test for reading from an invalid channel"""
with pytest.raises(expected_error):
with managed_env_var(cfg):
_get_reader(source="abc", batch_size=1000)
