def splice_fps(source_fp, dest_fp):
if not splice:
raise NotImplementedError('No splice() implementation')
logger.debug('Using splice_fps to copy file data')
pipe_r, pipe_w = os.pipe()
try:
fcntl.fcntl(pipe_w, F_SETPIPE_SZ, CHUNKSIZE)
except IOError:
logger.debug('Failed to expand pipe buffer to chunksize %d', CHUNKSIZE)
try:
while True:
# splice into kernel memory
buffered = splice.splice(
source_fp.fileno(), None, pipe_w, None, CHUNKSIZE,
splice.SPLICE_F_MOVE | splice.SPLICE_F_MORE)
if buffered == 0:
break # done
# splice into dest file
while buffered > 0:
ret = splice.splice(
pipe_r, None, dest_fp.fileno(), None, buffered,
splice.SPLICE_F_MOVE | splice.SPLICE_F_MORE)
if ret > 0:
buffered -= ret
except IOError as e:
if e.errno in (errno.EBADF, errno.EINVAL):
raise NotImplementedError('Unsupported splice() implementation')
def splice_fps(source_fp, dest_fp):
if not splice:
raise NotImplementedError('No splice() implementation')
logger.debug('Using splice_fps to copy file data')
pipe_r, pipe_w = os.pipe()
try:
fcntl.fcntl(pipe_w, F_SETPIPE_SZ, CHUNKSIZE)
except IOError:
logger.debug('Failed to expand pipe buffer to chunksize %d', CHUNKSIZE)
try:
while True:
# splice into kernel memory
buffered = splice.splice(source_fp.fileno(), None, pipe_w, None, CHUNKSIZE,
splice.SPLICE_F_MOVE | splice.SPLICE_F_MORE)
if buffered == 0:
break # done
# splice into dest file
while buffered > 0:
ret = splice.splice(pipe_r, None, dest_fp.fileno(), None, buffered,
splice.SPLICE_F_MOVE | splice.SPLICE_F_MORE)
if ret > 0:
buffered -= ret
except IOError as e:
if e.errno in (errno.EBADF, errno.EINVAL):
raise NotImplementedError('Unsupported splice() implementation')
def test_incomplete_arguments(create_files):
(file_in, file_out) = create_files
with pytest.raises(TypeError):
splice(file_out.fileno(), offset=0)
with pytest.raises(TypeError):
splice(file_in.fileno(), offset=0)
def test_invalid_argument_types(create_files):
(file_in, file_out) = create_files
with pytest.raises(TypeError):
splice('test', file_out.fileno(), 0, 0)
with pytest.raises(TypeError):
splice(file_in.fileno(), 'test', 0, 0)
def test_offset_overflow(create_files):
(file_in, file_out) = create_files
content_in = file_in.read()
offset = len(content_in) + 1
with pytest.raises(OverflowError):
splice(file_in.fileno(), file_out.fileno(), offset=offset)
def test_invalid_file_descriptors(create_files):
(file_in, file_out) = create_files
content_in = file_in.read()
with pytest.raises(ValueError):
splice(999, file_out.fileno(), 0, len(content_in))
with pytest.raises(ValueError):
splice(file_in.fileno(), 999, 0, len(content_in))
def test_small_file_with_offset_overflow(create_files):
(file_in, file_out) = create_files
# create small file
file_in = tempfile.TemporaryFile()
file_in.write(b'foo bar')
file_in.seek(0)
offset = 4096
with pytest.raises(OverflowError):
splice(file_in.fileno(), file_out.fileno(), offset=offset)
def main(port1, port2):
# connect to both endpoints
HOST = ''
ep1 = EndPoint(HOST, port1)
ep2 = EndPoint(HOST, port2)
# start both
ep1.start()
ep2.start()
# wait for both to connect
while not (ep1.event.is_set() and ep2.event.is_set()):
time.sleep(0.1)
# we're connected on both ends. Start splicing!
ep1_fd = ep1.sock.fileno()
set_nonblock(ep1_fd)
ep2_fd = ep2.sock.fileno()
set_nonblock(ep2_fd)
# get splicing
chunksize = 1024
flags = \
splice.SPLICE_F_MOVE | splice.SPLICE_F_MORE | splice.SPLICE_F_NONBLOCK
# run the transfer
while True:
# 2->1 direction
f1_read, f2_write, _ = select.select([ep1_fd], [ep2_fd], [])
if f1_read and f2_write:
# splice!
try:
d21 = splice.splice(ep2_fd, None, ep1_fd, None, chunksize, flags)
except IOError, exc:
if exc.errno in [errno.EAGAIN, errno.EWOULDBLOCK]:
# one of the fds blocked... retry!
pass
else:
raise
# 1->2 direction
f2_read, f1_write, _ = select.select([ep2_fd], [ep1_fd], [])
if f2_read and f1_write:
# splice!
try:
d12 = splice.splice(ep1_fd, None, ep2_fd, None, chunksize, flags)
except IOError, exc:
if exc.errno in [errno.EAGAIN, errno.EWOULDBLOCK]:
# one of the fds blocked... retry!
pass
else:
raise
def test_simple_file(create_files):
(file_in, file_out) = create_files
content_in = file_in.read()
bytes_copied = splice(file_in.fileno(), file_out.fileno())
content_out = file_out.read()
assert bytes_copied == len(content_in)
assert bytes_copied == len(content_out)
assert len(content_out) == len(content_in)
assert hash(content_out) == hash(content_in)
def test_copy_from_certain_offset(create_files):
(file_in, file_out) = create_files
content_in = file_in.read()
offset = 1024
bytes_copied = splice(file_in.fileno(), file_out.fileno(), offset=offset)
content_out = file_out.read()
assert bytes_copied == len(content_out)
assert bytes_copied == len(content_in[offset:])
assert len(content_in[offset:]) == len(content_out)
assert hash(content_in[offset:]) == hash(content_out)
def test_simple_file_with_all_flags(create_files):
(file_in, file_out) = create_files
content_in = file_in.read()
bytes_copied = splice(file_in.fileno(),
file_out.fileno(),
flags=SPLICE_F_MORE | SPLICE_F_MOVE
| SPLICE_F_NONBLOCK | SPLICE_F_GIFT)
content_out = file_out.read()
assert bytes_copied == len(content_in)
assert bytes_copied == len(content_out)
assert len(content_out) == len(content_in)
assert hash(content_out) == hash(content_in)
def _proc_write(command, outfile, limit, cleaner, stderr_logger):
proc = subprocess.Popen(command, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
proc_desc = '{} (pid {})'.format(
command if isinstance(command, str) else command[0], proc.pid)
proc_stdout = proc.stdout.fileno()
stderr_logger.startLogging(proc.stderr, proc_desc)
total_num_written = 0
num_splice_calls = 0
try:
while True:
if limit <= 0:
logging.debug('Ran out of space: {} bytes left'.format(limit))
limit += cleaner()
logging.debug('After cleanup, new capacity is {} bytes'
.format(limit))
try:
num_splice_calls += 1
eff_limit = min(limit, 2**30) # avoid int overflows
num_written = splice(proc_stdout, outfile, eff_limit)
except OSError as e:
logging.error(exc_str(e))
status = None
logging.info('Waiting for {} to exit'.format(proc_desc))
try:
status = proc.wait(_KILL_TIMEOUT_SECS)
logging.debug('{} exited with status {}'.format(proc_desc,
status))
except subprocess.TimeoutExpired:
logging.warning('{} timed out: terminating'.format(
proc_desc))
_kill(proc, proc_desc)
return status, total_num_written
total_num_written += num_written
limit -= num_written
if num_written == 0:
logging.debug('Finished: wrote {} bytes in {} splice() '
'calls; waiting for {} to exit'.format(
total_num_written, num_splice_calls,
proc_desc))
status = proc.wait()
logging.debug('{} exited with status {}'.format(proc_desc,
status))
return status, total_num_written
except:
logging.warning('Terminating {}'.format(proc_desc))
_kill(proc, proc_desc)
raise
def test_small_file(create_files):
(file_in, file_out) = create_files
# create small file
file_in = tempfile.TemporaryFile()
file_in.write(b'foo bar')
file_in.seek(0)
content_in = file_in.read()
bytes_copied = splice(file_in.fileno(), file_out.fileno())
content_out = file_out.read()
assert bytes_copied == len(content_in)
assert bytes_copied == len(content_out)
assert len(content_out) == len(content_in)
assert hash(content_in) == hash(content_out)
def test_simple_file_with_all_optional_args(create_files):
(file_in, file_out) = create_files
content_in = file_in.read()
bytes_copied = splice(file_in.fileno(),
file_out.fileno(),
offset=0,
nbytes=len(content_in),
flags=SPLICE_F_MORE)
content_out = file_out.read()
assert bytes_copied == len(content_in)
assert bytes_copied == len(content_out)
assert len(content_out) == len(content_in)
assert hash(content_out) == hash(content_in)
def test_copy_certain_bytes_copied(create_files):
(file_in, file_out) = create_files
content_in = file_in.read()
bytes_to_copy = 2048
bytes_copied = splice(file_in.fileno(),
file_out.fileno(),
nbytes=bytes_to_copy)
content_out = file_out.read()
assert bytes_copied == bytes_to_copy
assert bytes_copied == len(content_out)
assert bytes_to_copy == len(content_out)
assert len(content_in[:bytes_to_copy]) == len(content_out)
assert hash(content_in[:bytes_to_copy]) == hash(content_out)
def test_large_file(create_files):
(file_in, file_out) = create_files
# create a large file
file_in = tempfile.SpooledTemporaryFile()
file_in.write(SAMPLE_DATA * 1) # * 1024 when automating tests (~1GB)
file_in.seek(0)
content_in = file_in.read()
bytes_copied = splice(file_in.fileno(),
file_out.fileno(),
nbytes=len(content_in))
content_out = file_out.read()
assert bytes_copied == len(content_in)
assert bytes_copied == len(content_out)
assert len(content_in) == len(content_out)
assert hash(content_in) == hash(content_out)
def splice_copy():
t0 = pc()
f1 = open('read.txt')
f2 = open('write2.txt', 'w+')
splice.splice(f1.fileno(), f2.fileno())
return (pc() - t0)
def test_len_overflow(create_files):
(file_in, file_out) = create_files
content_in = file_in.read()
with pytest.raises(OverflowError):
splice(file_in.fileno(), file_out.fileno(), nbytes=len(content_in) + 1)
