def test_probe_block_size_unsupported():
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
# /dev/shm uses tempfs, does not support direct I/O.
with pytest.raises(OSError) as e:
proc.probe_block_size("/dev/shm")
assert e.value.errno == errno.EINVAL
def testEcho(self):
data = """The Doctor: But I don't exist in your world!
Brigade Leader: Then you won't feel the bullets when we
shoot you.""" # (C) BBC - Doctor Who
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
self.assertEquals(proc.echo(data), data)
def testStatvfs(self):
data = b'''Peter Puppy: Once again, evil is as rotting meat before
the maggots of justice!
Earthworm Jim: Thank you for cramming that delightful image
into my brain, Peter.
''' # (C) Universal Cartoon Studios - Earth Worm Jim
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
fd, path = mkstemp()
try:
os.write(fd, data)
os.close(fd)
pystat = os.statvfs(path)
mystat = proc.statvfs(path)
for f in ("f_bsize", "f_frsize", "f_blocks", "f_fsid",
"f_flag", "f_namemax"):
try:
getattr(pystat, f)
except AttributeError:
# The results might be more comprehansive then python
# implementation
continue
log.debug("Testing field '%s'", f)
self.assertEquals(getattr(mystat, f), getattr(pystat, f))
finally:
os.unlink(path)
def test_lstat_link_missing(tmpdir):
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
src = str(tmpdir.join("file"))
link = str(tmpdir.join("link"))
os.symlink(src, link)
check_stat(proc.lstat(link), os.lstat(link))
def testNonExistingMethod(self):
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
try:
proc._sendCommand("Implode", {}, proc.timeout)
except OSError as e:
self.assertEquals(e.errno, errno.EINVAL)
def test_readfile_missing(tmpdir, direct):
path = str(tmpdir.join("file"))
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
with pytest.raises(OSError) as e:
read = proc.readfile(path, direct=direct)
assert e.value.errno == errno.ENOENT
def testEcho(self):
data = """The Doctor: But I don't exist in your world!
Brigade Leader: Then you won't feel the bullets when we
shoot you.""" # (C) BBC - Doctor Who
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
self.assertEquals(proc.echo(data), data)
def test_lstat_file(tmpdir):
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
file = tmpdir.join("file")
file.write(b"x" * 100)
file = str(file)
check_stat(proc.lstat(file), os.lstat(file))
def testMissingArguemt(self):
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
try:
proc._sendCommand("echo", {}, proc.timeout)
except OSError as e:
self.assertEquals(e.errno, errno.EINVAL)
def test_lstat_file(tmpdir):
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
file = tmpdir.join("file")
file.write(b"x" * 100)
file = str(file)
check_stat(proc.lstat(file), os.lstat(file))
def testNonExistingMethod(self):
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
try:
proc._sendCommand("Implode", {}, proc.timeout)
except OSError as e:
self.assertEquals(e.errno, errno.EINVAL)
def testMissingArguemt(self):
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
try:
proc._sendCommand("echo", {}, proc.timeout)
except OSError as e:
self.assertEquals(e.errno, errno.EINVAL)
def test_lstat_link_missing(tmpdir):
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
src = str(tmpdir.join("file"))
link = str(tmpdir.join("link"))
os.symlink(src, link)
check_stat(proc.lstat(link), os.lstat(link))
def testStatvfs(self):
data = b'''Peter Puppy: Once again, evil is as rotting meat before
the maggots of justice!
Earthworm Jim: Thank you for cramming that delightful image
into my brain, Peter.
''' # (C) Universal Cartoon Studios - Earth Worm Jim
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
fd, path = mkstemp()
try:
os.write(fd, data)
os.close(fd)
pystat = os.statvfs(path)
mystat = proc.statvfs(path)
for f in ("f_bsize", "f_frsize", "f_blocks",
"f_fsid", "f_flag", "f_namemax"):
try:
getattr(pystat, f)
except AttributeError:
# The results might be more comprehansive then python
# implementation
continue
log.debug("Testing field '%s'", f)
self.assertEquals(getattr(mystat, f), getattr(pystat, f))
finally:
os.unlink(path)
def test_fsyncpath_file(tmpdir):
proc = IOProcess(timeout=1, max_threads=1)
with closing(proc):
path = tmpdir.join("file")
path.write("data")
# No easy way to test that we actually fsync this path. Lets just
# call it to make sure it does not fail.
assert proc.fsyncPath(str(path)) is None
def test_writefile_direct_unaligned(tmpdir):
data = b'unaligned data'
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
path = str(tmpdir.join("file"))
with pytest.raises(OSError) as e:
proc.writefile(path, data, direct=True)
assert e.value.errno == errno.EINVAL
def test_fsyncpath_file(tmpdir):
proc = IOProcess(timeout=1, max_threads=1)
with closing(proc):
path = tmpdir.join("file")
path.write("data")
# No easy way to test that we actually fsync this path. Lets just
# call it to make sure it does not fail.
assert proc.fsyncPath(str(path)) is None
def test_writefile_direct_aligned(tmpdir, size):
data = b'x' * size
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
path = str(tmpdir.join("file"))
proc.writefile(path, data, direct=True)
with io.open(path, 'rb') as f:
written = f.read()
assert written == data
def testAccess(self):
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
fd, path = mkstemp()
try:
os.close(fd)
self.assertTrue(proc.access(path, os.W_OK))
finally:
os.unlink(path)
def testPathExists(self):
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
fd, path = mkstemp()
try:
os.close(fd)
self.assertTrue(proc.pathExists(path))
finally:
os.unlink(path)
def test_access_directory(tmpdir, mode, permission, expected_result):
proc = IOProcess(timeout=10, max_threads=5)
d = tmpdir.mkdir("subdir")
path = str(d)
with closing(proc):
with chmod(path, mode):
assert proc.access(path, permission) == os.access(path, permission)
assert proc.access(path, permission) == expected_result
def test_probe_block_size_not_writable(tmpdir):
no_write = str(tmpdir.mkdir("no-write-for-you"))
# Remove write bit, so the probe file cannot be created.
with chmod(no_write, 0o500):
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
with pytest.raises(OSError) as e:
proc.probe_block_size(no_write)
assert e.value.errno == errno.EACCES
def test_lstat_link(tmpdir):
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
src = tmpdir.join("src")
src.write(b"x" * 100)
src = str(src)
link = str(tmpdir.join("link"))
os.symlink(src, link)
check_stat(proc.lstat(link), os.lstat(link))
def testAccess(self):
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
fd, path = mkstemp()
try:
os.close(fd)
self.assertTrue(proc.access(path, os.W_OK))
finally:
os.unlink(path)
def test_lstat_link(tmpdir):
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
src = tmpdir.join("src")
src.write(b"x" * 100)
src = str(src)
link = str(tmpdir.join("link"))
os.symlink(src, link)
check_stat(proc.lstat(link), os.lstat(link))
def test_readfile(tmpdir, size):
data = b'x' * size
path = str(tmpdir.join("file"))
with io.open(path, "wb") as f:
f.write(data)
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
read = proc.readfile(path)
assert read == data
def testChmodFail(self):
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
try:
proc.chmod("/I/do/not/exist", 0)
except OSError as e:
if e.errno != errno.ENOENT:
raise
else:
raise AssertionError("OSError was not raised")
def testStatFail(self):
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
try:
proc.stat("/I do not exist")
except OSError as e:
if e.errno != errno.ENOENT:
raise
else:
raise AssertionError("OSError was not raised")
def testSymlinkFail(self):
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
try:
proc.symlink("/Dsadsad", "/I/do/not/exist")
except OSError as e:
if e.errno != errno.ENOENT:
raise
else:
raise AssertionError("OSError was not raised")
def test_access_file(tmpdir, mode, permission, expected_result):
proc = IOProcess(timeout=10, max_threads=5)
f = tmpdir.join("file")
f.write("")
path = str(f)
with closing(proc):
with chmod(path, mode):
assert proc.access(path, permission) == os.access(path, permission)
assert proc.access(path, permission) == expected_result
def testSymlinkFail(self):
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
try:
proc.symlink("/Dsadsad", "/I/do/not/exist")
except OSError as e:
if e.errno != errno.ENOENT:
raise
else:
raise AssertionError("OSError was not raised")
def testChmodFail(self):
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
try:
proc.chmod("/I/do/not/exist", 0)
except OSError as e:
if e.errno != errno.ENOENT:
raise
else:
raise AssertionError("OSError was not raised")
def testStatFail(self):
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
try:
proc.stat("/I do not exist")
except OSError as e:
if e.errno != errno.ENOENT:
raise
else:
raise AssertionError("OSError was not raised")
def testRecoverAfterCrash(self):
data = """Brigadier: Is there anything I can do?
Third Doctor: Yes, pass me a silicon rod.
[Stirs cup of tea with it]
Brigadier: I meant is there anything UNIT can do about this
space lightning business?""" # (C) BBC - Doctor Who
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
self.assertEquals(proc.echo(data), data)
self.assertTrue(proc.crash())
self.assertEquals(proc.echo(data), data)
def test_stat_link_missing(tmpdir):
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
src = str(tmpdir.join("file"))
link = str(tmpdir.join("link"))
os.symlink(src, link)
with pytest.raises(OSError) as myerror:
proc.stat(link)
with pytest.raises(OSError) as pyerror:
os.stat(link)
assert myerror.value.errno == pyerror.value.errno
def test_stat_link_missing(tmpdir):
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
src = str(tmpdir.join("file"))
link = str(tmpdir.join("link"))
os.symlink(src, link)
with pytest.raises(OSError) as myerror:
proc.stat(link)
with pytest.raises(OSError) as pyerror:
os.stat(link)
assert myerror.value.errno == pyerror.value.errno
def testRecoverAfterCrash(self):
data = """Brigadier: Is there anything I can do?
Third Doctor: Yes, pass me a silicon rod.
[Stirs cup of tea with it]
Brigadier: I meant is there anything UNIT can do about this
space lightning business?""" # (C) BBC - Doctor Who
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
self.assertEquals(proc.echo(data), data)
self.assertTrue(proc.crash())
self.assertEquals(proc.echo(data), data)
def test_close_unrelated_fds(self):
# Make inheritable file descriptor.
with open(__file__) as my_file:
clear_cloexec(my_file.fileno())
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
# Wait until ready.
proc.ping()
proc_fd = "/proc/%d/fd" % proc.pid
child_fds = [int(fd) for fd in os.listdir(proc_fd)]
# My file descriptor must not be inherited.
self.assertNotIn(my_file.fileno(), child_fds)
def testUnlink(self):
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
fd, path = mkstemp()
try:
os.close(fd)
self.assertTrue(proc.unlink(path))
finally:
try:
os.unlink(path)
except:
pass
def test_close_unrelated_fds(self):
# Make inheritable file descriptor.
with open(__file__) as my_file:
clear_cloexec(my_file.fileno())
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
# Wait until ready.
proc.ping()
proc_fd = "/proc/%d/fd" % proc.pid
child_fds = [int(fd) for fd in os.listdir(proc_fd)]
# My file descriptor must not be inherited.
self.assertNotIn(my_file.fileno(), child_fds)
def testUnlink(self):
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
fd, path = mkstemp()
try:
os.close(fd)
self.assertTrue(proc.unlink(path))
finally:
try:
os.unlink(path)
except:
pass
def testLexists(self):
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
path = "/tmp/linktest.ioprocesstest"
try:
os.unlink(path)
except OSError:
pass
os.symlink("dsadsadsadsad", path)
try:
self.assertTrue(proc.lexists(path))
finally:
os.unlink(path)
def testRmdir(self):
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
path = mkdtemp()
try:
proc.rmdir(path)
self.assertFalse(os.path.exists(path))
finally:
try:
shutil.rmtree(path)
except:
pass
def testTimeout(self):
data = """Madge: Are you the new caretaker?
The Doctor: Usually called "The Doctor." Or "The Caretaker."
Or "Get off this planet." Though, strictly speaking, that
probably isn't a name.""" # (C) BBC - Doctor Who
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
try:
self.assertEquals(proc.echo(data, 10), data)
except Timeout:
return
self.fail("Exception not raised")
def testChmod(self):
proc = IOProcess(timeout=10, max_threads=5)
with closing(proc):
fd, path = mkstemp()
targetMode = os.W_OK | os.R_OK
try:
os.chmod(path, 0)
os.close(fd)
self.assertFalse(os.stat(path).st_mode & targetMode)
self.assertTrue(proc.chmod(path, targetMode))
self.assertTrue(os.stat(path).st_mode & targetMode)
finally:
os.unlink(path)
def testChmod(self):
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
fd, path = mkstemp()
targetMode = os.W_OK | os.R_OK
try:
os.chmod(path, 0)
os.close(fd)
self.assertFalse(os.stat(path).st_mode & targetMode)
self.assertTrue(proc.chmod(path, targetMode))
self.assertTrue(os.stat(path).st_mode & targetMode)
finally:
os.unlink(path)
def testRmdir(self):
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
path = mkdtemp()
try:
proc.rmdir(path)
self.assertFalse(os.path.exists(path))
finally:
try:
shutil.rmtree(path)
except:
pass
def testLexists(self):
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
path = "/tmp/linktest.ioprocesstest"
try:
os.unlink(path)
except OSError:
pass
os.symlink("dsadsadsadsad", path)
try:
self.assertTrue(proc.lexists(path))
finally:
os.unlink(path)
def test_probe_block_size_not_writable(tmpdir):
no_write = str(tmpdir.mkdir("no-write-for-you"))
# Remove write bit, so the probe file cannot be created.
with chmod(no_write, 0o500):
proc = IOProcess(timeout=10, max_threads=5)
proc._sublog = FakeLogger()
with closing(proc):
with pytest.raises(OSError) as e:
proc.probe_block_size(no_write)
assert e.value.errno == errno.EACCES
warning = re.compile(r"Failed to create a probe file: .+?, "
r"error: 'Permission denied'")
assert any(warning.search(msg) for msg in proc._sublog.messages)
def testTimeout(self):
data = """Madge: Are you the new caretaker?
The Doctor: Usually called "The Doctor." Or "The Caretaker."
Or "Get off this planet." Though, strictly speaking, that
probably isn't a name.""" # (C) BBC - Doctor Who
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
try:
self.assertEquals(proc.echo(data, 10), data)
except Timeout:
return
self.fail("Exception not raised")
def testLink(self):
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
fd, oldpath = mkstemp()
newpath = oldpath + ".new"
try:
os.close(fd)
self.assertTrue(proc.link(oldpath, newpath))
finally:
os.unlink(oldpath)
try:
os.unlink(newpath)
except:
pass
def testSymlink(self):
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
fd, oldpath = mkstemp()
newpath = oldpath + ".new"
try:
os.close(fd)
self.assertTrue(proc.symlink(oldpath, newpath))
self.assertEquals(os.path.realpath(newpath),
os.path.normpath(oldpath))
finally:
os.unlink(oldpath)
try:
os.unlink(newpath)
except:
pass
def testGlob(self):
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
path = mkdtemp()
matches = []
for i in range(10):
matches.append(os.path.join(path, str(i)))
with open(matches[-1], "w") as f:
f.write("A")
matches.sort()
try:
remoteMatches = proc.glob(os.path.join(path, "*"))
remoteMatches.sort()
self.assertEquals(remoteMatches, matches)
finally:
shutil.rmtree(path)
def testReadfile(self, direct=False):
data = b'''The Doctor: Well... you could do that. Yeah, you could do
that. Of course you could! But why? Look at these people,
these human beings. Consider their potential! From the day
they arrive on the planet, blinking, step into the sun,
there is more to see than can ever be seen, more to do
than-no, hold on. Sorry, that's The Lion King.
But the point still stands: leave them alone!
''' # (C) BBC - Doctor Who
proc = IOProcess(timeout=1, max_threads=5)
with closing(proc):
fd, path = mkstemp(dir="/var/tmp")
try:
os.write(fd, data)
os.close(fd)
remoteData = proc.readfile(path, direct)
self.assertEquals(remoteData[:len(data)], data)
finally:
os.unlink(path)
def testWritefile(self, direct=False):
data = b'''Jackie: I'm in my dressing gown.
The Doctor: Yes, you are.
Jackie: There's a strange man in my bedroom.
The Doctor: Yes, there is.
Jackie: Anything could happen.
The Doctor: No. [walks away]''' # (C) BBC - Doctor Who
# This test sometimes time out in the CI. On one failure, the write
# took 1.8 seconds inside ioprocess.
proc = IOProcess(timeout=5, max_threads=5)
with closing(proc):
fd, path = mkstemp(dir="/var/tmp")
try:
os.close(fd)
proc.writefile(path, data, direct)
with open(path, 'rb') as f:
diskData = f.read()
self.assertEquals(diskData[:len(data)], data)
finally:
os.unlink(path)
def testMaxRequests(self):
proc = IOProcess(timeout=5, max_threads=1, max_queued_requests=1)
with closing(proc):
t1 = Thread(target=proc.echo, args=("hello", 2))
t2 = Thread(target=proc.echo, args=("hello", 2))
t1.start()
t2.start()
# Make sure the echo calls are sent prior to the ping otherwise one
# of them would fail and ping() would pass
time.sleep(0.5)
try:
proc.ping()
except OSError as e:
self.assertEquals(e.errno, errno.EAGAIN)
except Exception:
self.fail("Expected OSError got %s", e)
else:
self.fail("Expected exception")
finally:
t1.join()
t2.join()
def testMaxRequestsAfterFillingThreadPool(self):
proc = IOProcess(timeout=5, max_threads=3, max_queued_requests=0)
with closing(proc):
t1 = Thread(target=proc.echo, args=("hello", 2))
t2 = Thread(target=proc.echo, args=("hello", 2))
t3 = Thread(target=proc.echo, args=("hello", 2))
t1.start()
t2.start()
t3.start()
for t in (t1, t2, t3):
t.join()
t1 = Thread(target=proc.echo, args=("hello", 2))
t2 = Thread(target=proc.echo, args=("hello", 2))
t1.start()
t2.start()
# Make sure the echo calls are sent prior to the ping otherwise one
# of them would fail and ping() would pass
time.sleep(0.5)
proc.ping()
t1.join()
t2.join()
def test_partial_logs(self):
threads = []
proc = IOProcess(timeout=1, max_threads=10)
proc._sublog = FakeLogger()
def worker():
for i in range(100):
proc.stat(__file__)
try:
for i in range(4):
t = Thread(target=worker)
t.deamon = True
t.start()
threads.append(t)
finally:
for t in threads:
t.join()
proc.close()
for msg in proc._sublog.messages:
self.assertFalse('DEBUG|' in msg,
"Raw log data in log message: %r" % msg)
def testPendingRequestInvalidationOnCrash(self):
data = """The Doctor: A straight line may be the shortest distance
between two points, but it is by no means the most
interesting.""" # (C) BBC - Doctor Who
proc = IOProcess(timeout=12, max_threads=5)
with closing(proc):
res = [False]
def sendCmd():
try:
proc.echo(data, 10)
except OSError as e:
if e.errno == ERR_IOPROCESS_CRASH:
res[0] = True
else:
log.error("Got unexpected error", exc_info=True)
t = Thread(target=sendCmd)
t.start()
time.sleep(1)
proc.crash()
t.join()
self.assertTrue(res[0])
def testManyRequests(self):
data = """Lily: What's happening?
The Doctor: No idea. Just do what I do: hold tight and
pretend it's a plan.""" # (C) BBC - Doctor Who
proc = IOProcess(timeout=30, max_threads=5)
with closing(proc):
# even though we theoretically go back to a stable state, some
# objects might have increased their internal buffers and mem
# fragmantation might have caused some data to be spanned on more
# pages then it originally did.
acceptableRSSIncreasKB = 100
startRSS = proc.memstat()['rss']
# This way we catch evey leak that is more then one 0.1KB per call
many = 300
for i in range(many):
self.assertEquals(proc.echo(data), data)
endRSS = proc.memstat()['rss']
RSSDiff = endRSS - startRSS
log.debug("RSS difference was %d KB, %d per request", RSSDiff,
RSSDiff / many)
# This only tests for leaks in the main request\response process.
self.assertTrue(RSSDiff < acceptableRSSIncreasKB,
"Detected a leak sized %d KB" % RSSDiff)
