def test_memory_map_close_remove(tmpdir):
# ARROW-6740: should be able to delete closed memory-mapped file (Windows)
path = os.path.join(str(tmpdir), guid())
mmap = pa.create_memory_map(path, 4096)
mmap.close()
assert mmap.closed
os.remove(path) # Shouldn't fail
def test_tensor_ipc_strided(tmpdir):
data = np.random.randn(10, 4)
tensor = pa.Tensor.from_numpy(data[::2])
path = os.path.join(str(tmpdir), 'pyarrow-tensor-ipc-strided')
with pytest.raises(ValueError):
mmap = pa.create_memory_map(path, 1024)
pa.write_tensor(tensor, mmap)
def test_tensor_ipc_strided(tmpdir):
data = np.random.randn(10, 4)
tensor = pa.Tensor.from_numpy(data[::2])
path = os.path.join(str(tmpdir), 'pyarrow-tensor-ipc-strided')
with pytest.raises(ValueError):
mmap = pa.create_memory_map(path, 1024)
pa.write_tensor(tensor, mmap)
def test_tensor_ipc_strided():
data = np.random.randn(10, 4)
tensor = pa.Tensor.from_numpy(data[::2])
path = 'pyarrow-tensor-ipc-strided'
try:
with pytest.raises(ValueError):
mmap = pa.create_memory_map(path, 1024)
pa.write_tensor(tensor, mmap)
finally:
_try_delete(path)
def test_read_tensor(tmpdir):
# Create and write tensor tensor
data = np.random.randn(10, 4)
tensor = pa.Tensor.from_numpy(data)
data_size = pa.get_tensor_size(tensor)
path = os.path.join(str(tmpdir), 'pyarrow-tensor-ipc-read-tensor')
write_mmap = pa.create_memory_map(path, data_size)
pa.write_tensor(tensor, write_mmap)
# Try to read tensor
read_mmap = pa.memory_map(path, mode='r')
array = pa.read_tensor(read_mmap).to_numpy()
np.testing.assert_equal(data, array)
def test_read_tensor(tmpdir):
# Create and write tensor tensor
data = np.random.randn(10, 4)
tensor = pa.Tensor.from_numpy(data)
data_size = pa.get_tensor_size(tensor)
path = os.path.join(str(tmpdir), 'pyarrow-tensor-ipc-read-tensor')
write_mmap = pa.create_memory_map(path, data_size)
pa.write_tensor(tensor, write_mmap)
# Try to read tensor
read_mmap = pa.memory_map(path, mode='r')
array = pa.read_tensor(read_mmap).to_numpy()
np.testing.assert_equal(data, array)
def test_tensor_ipc_roundtrip(tmpdir):
data = np.random.randn(10, 4)
tensor = pa.Tensor.from_numpy(data)
path = os.path.join(str(tmpdir), 'pyarrow-tensor-ipc-roundtrip')
mmap = pa.create_memory_map(path, 1024)
pa.write_tensor(tensor, mmap)
mmap.seek(0)
result = pa.read_tensor(mmap)
assert result.equals(tensor)
def test_tensor_ipc_roundtrip(tmpdir):
data = np.random.randn(10, 4)
tensor = pa.Tensor.from_numpy(data)
path = os.path.join(str(tmpdir), 'pyarrow-tensor-ipc-roundtrip')
mmap = pa.create_memory_map(path, 1024)
pa.write_tensor(tensor, mmap)
mmap.seek(0)
result = pa.read_tensor(mmap)
assert result.equals(tensor)
def test_tensor_ipc_roundtrip():
data = np.random.randn(10, 4)
tensor = pa.Tensor.from_numpy(data)
path = 'pyarrow-tensor-ipc-roundtrip'
try:
mmap = pa.create_memory_map(path, 1024)
pa.write_tensor(tensor, mmap)
mmap.seek(0)
result = pa.read_tensor(mmap)
assert result.equals(tensor)
finally:
_try_delete(path)
def test_memory_map_resize(tmpdir):
SIZE = 4096
arr = np.random.randint(0, 256, size=SIZE).astype(np.uint8)
data1 = arr.tobytes()[:(SIZE // 2)]
data2 = arr.tobytes()[(SIZE // 2):]
path = os.path.join(str(tmpdir), guid())
mmap = pa.create_memory_map(path, SIZE / 2)
mmap.write(data1)
mmap.resize(SIZE)
mmap.write(data2)
mmap.close()
with open(path, 'rb') as f:
assert f.read() == arr.tobytes()
def test_tensor_ipc_strided(tmpdir):
data1 = np.random.randn(10, 4)
tensor1 = pa.Tensor.from_numpy(data1[::2])
data2 = np.random.randn(10, 6, 4)
tensor2 = pa.Tensor.from_numpy(data2[::, ::2, ::])
path = os.path.join(str(tmpdir), 'pyarrow-tensor-ipc-strided')
mmap = pa.create_memory_map(path, 2048)
for tensor in [tensor1, tensor2]:
mmap.seek(0)
pa.write_tensor(tensor, mmap)
mmap.seek(0)
result = pa.read_tensor(mmap)
assert result.equals(tensor)
def test_tensor_ipc_strided(tmpdir):
data1 = np.random.randn(10, 4)
tensor1 = pa.Tensor.from_numpy(data1[::2])
data2 = np.random.randn(10, 6, 4)
tensor2 = pa.Tensor.from_numpy(data2[::, ::2, ::])
path = os.path.join(str(tmpdir), 'pyarrow-tensor-ipc-strided')
mmap = pa.create_memory_map(path, 2048)
for tensor in [tensor1, tensor2]:
mmap.seek(0)
pa.write_tensor(tensor, mmap)
mmap.seek(0)
result = pa.read_tensor(mmap)
assert result.equals(tensor)
def test_memory_map_resize(tmpdir):
SIZE = 4096
arr = np.random.randint(0, 256, size=SIZE).astype(np.uint8)
data1 = arr.tobytes()[:(SIZE // 2)]
data2 = arr.tobytes()[(SIZE // 2):]
path = os.path.join(str(tmpdir), guid())
mmap = pa.create_memory_map(path, SIZE / 2)
mmap.write(data1)
mmap.resize(SIZE)
mmap.write(data2)
mmap.close()
with open(path, 'rb') as f:
assert f.read() == arr.tobytes()
def test_memory_map_deref_remove(tmpdir):
path = os.path.join(str(tmpdir), guid())
pa.create_memory_map(path, 4096)
os.remove(path) # Shouldn't fail
