def test_probe_block_size_unsupported(): proc = IOProcess(timeout=10, max_threads=5) with closing(proc): # /dev/shm uses tempfs, does not support direct I/O. with pytest.raises(OSError) as e: proc.probe_block_size("/dev/shm") assert e.value.errno == errno.EINVAL
def testEcho(self): data = """The Doctor: But I don't exist in your world! Brigade Leader: Then you won't feel the bullets when we shoot you.""" # (C) BBC - Doctor Who proc = IOProcess(timeout=1, max_threads=5) with closing(proc): self.assertEquals(proc.echo(data), data)
def testStatvfs(self): data = b'''Peter Puppy: Once again, evil is as rotting meat before the maggots of justice! Earthworm Jim: Thank you for cramming that delightful image into my brain, Peter. ''' # (C) Universal Cartoon Studios - Earth Worm Jim proc = IOProcess(timeout=10, max_threads=5) with closing(proc): fd, path = mkstemp() try: os.write(fd, data) os.close(fd) pystat = os.statvfs(path) mystat = proc.statvfs(path) for f in ("f_bsize", "f_frsize", "f_blocks", "f_fsid", "f_flag", "f_namemax"): try: getattr(pystat, f) except AttributeError: # The results might be more comprehansive then python # implementation continue log.debug("Testing field '%s'", f) self.assertEquals(getattr(mystat, f), getattr(pystat, f)) finally: os.unlink(path)
def test_lstat_link_missing(tmpdir): proc = IOProcess(timeout=1, max_threads=5) with closing(proc): src = str(tmpdir.join("file")) link = str(tmpdir.join("link")) os.symlink(src, link) check_stat(proc.lstat(link), os.lstat(link))
def testNonExistingMethod(self): proc = IOProcess(timeout=10, max_threads=5) with closing(proc): try: proc._sendCommand("Implode", {}, proc.timeout) except OSError as e: self.assertEquals(e.errno, errno.EINVAL)
def test_readfile_missing(tmpdir, direct): path = str(tmpdir.join("file")) proc = IOProcess(timeout=10, max_threads=5) with closing(proc): with pytest.raises(OSError) as e: read = proc.readfile(path, direct=direct) assert e.value.errno == errno.ENOENT
def testEcho(self): data = """The Doctor: But I don't exist in your world! Brigade Leader: Then you won't feel the bullets when we shoot you.""" # (C) BBC - Doctor Who proc = IOProcess(timeout=10, max_threads=5) with closing(proc): self.assertEquals(proc.echo(data), data)
def test_lstat_file(tmpdir): proc = IOProcess(timeout=10, max_threads=5) with closing(proc): file = tmpdir.join("file") file.write(b"x" * 100) file = str(file) check_stat(proc.lstat(file), os.lstat(file))
def testMissingArguemt(self): proc = IOProcess(timeout=10, max_threads=5) with closing(proc): try: proc._sendCommand("echo", {}, proc.timeout) except OSError as e: self.assertEquals(e.errno, errno.EINVAL)
def test_lstat_file(tmpdir): proc = IOProcess(timeout=1, max_threads=5) with closing(proc): file = tmpdir.join("file") file.write(b"x" * 100) file = str(file) check_stat(proc.lstat(file), os.lstat(file))
def testNonExistingMethod(self): proc = IOProcess(timeout=1, max_threads=5) with closing(proc): try: proc._sendCommand("Implode", {}, proc.timeout) except OSError as e: self.assertEquals(e.errno, errno.EINVAL)
def testMissingArguemt(self): proc = IOProcess(timeout=1, max_threads=5) with closing(proc): try: proc._sendCommand("echo", {}, proc.timeout) except OSError as e: self.assertEquals(e.errno, errno.EINVAL)
def test_lstat_link_missing(tmpdir): proc = IOProcess(timeout=10, max_threads=5) with closing(proc): src = str(tmpdir.join("file")) link = str(tmpdir.join("link")) os.symlink(src, link) check_stat(proc.lstat(link), os.lstat(link))
def testStatvfs(self): data = b'''Peter Puppy: Once again, evil is as rotting meat before the maggots of justice! Earthworm Jim: Thank you for cramming that delightful image into my brain, Peter. ''' # (C) Universal Cartoon Studios - Earth Worm Jim proc = IOProcess(timeout=1, max_threads=5) with closing(proc): fd, path = mkstemp() try: os.write(fd, data) os.close(fd) pystat = os.statvfs(path) mystat = proc.statvfs(path) for f in ("f_bsize", "f_frsize", "f_blocks", "f_fsid", "f_flag", "f_namemax"): try: getattr(pystat, f) except AttributeError: # The results might be more comprehansive then python # implementation continue log.debug("Testing field '%s'", f) self.assertEquals(getattr(mystat, f), getattr(pystat, f)) finally: os.unlink(path)
def test_fsyncpath_file(tmpdir): proc = IOProcess(timeout=1, max_threads=1) with closing(proc): path = tmpdir.join("file") path.write("data") # No easy way to test that we actually fsync this path. Lets just # call it to make sure it does not fail. assert proc.fsyncPath(str(path)) is None
def test_writefile_direct_unaligned(tmpdir): data = b'unaligned data' proc = IOProcess(timeout=10, max_threads=5) with closing(proc): path = str(tmpdir.join("file")) with pytest.raises(OSError) as e: proc.writefile(path, data, direct=True) assert e.value.errno == errno.EINVAL
def test_fsyncpath_file(tmpdir): proc = IOProcess(timeout=1, max_threads=1) with closing(proc): path = tmpdir.join("file") path.write("data") # No easy way to test that we actually fsync this path. Lets just # call it to make sure it does not fail. assert proc.fsyncPath(str(path)) is None
def test_writefile_direct_aligned(tmpdir, size): data = b'x' * size proc = IOProcess(timeout=10, max_threads=5) with closing(proc): path = str(tmpdir.join("file")) proc.writefile(path, data, direct=True) with io.open(path, 'rb') as f: written = f.read() assert written == data
def testAccess(self): proc = IOProcess(timeout=1, max_threads=5) with closing(proc): fd, path = mkstemp() try: os.close(fd) self.assertTrue(proc.access(path, os.W_OK)) finally: os.unlink(path)
def testPathExists(self): proc = IOProcess(timeout=10, max_threads=5) with closing(proc): fd, path = mkstemp() try: os.close(fd) self.assertTrue(proc.pathExists(path)) finally: os.unlink(path)
def test_access_directory(tmpdir, mode, permission, expected_result): proc = IOProcess(timeout=10, max_threads=5) d = tmpdir.mkdir("subdir") path = str(d) with closing(proc): with chmod(path, mode): assert proc.access(path, permission) == os.access(path, permission) assert proc.access(path, permission) == expected_result
def test_probe_block_size_not_writable(tmpdir): no_write = str(tmpdir.mkdir("no-write-for-you")) # Remove write bit, so the probe file cannot be created. with chmod(no_write, 0o500): proc = IOProcess(timeout=10, max_threads=5) with closing(proc): with pytest.raises(OSError) as e: proc.probe_block_size(no_write) assert e.value.errno == errno.EACCES
def test_lstat_link(tmpdir): proc = IOProcess(timeout=10, max_threads=5) with closing(proc): src = tmpdir.join("src") src.write(b"x" * 100) src = str(src) link = str(tmpdir.join("link")) os.symlink(src, link) check_stat(proc.lstat(link), os.lstat(link))
def testAccess(self): proc = IOProcess(timeout=1, max_threads=5) with closing(proc): fd, path = mkstemp() try: os.close(fd) self.assertTrue(proc.access(path, os.W_OK)) finally: os.unlink(path)
def test_lstat_link(tmpdir): proc = IOProcess(timeout=1, max_threads=5) with closing(proc): src = tmpdir.join("src") src.write(b"x" * 100) src = str(src) link = str(tmpdir.join("link")) os.symlink(src, link) check_stat(proc.lstat(link), os.lstat(link))
def test_readfile(tmpdir, size): data = b'x' * size path = str(tmpdir.join("file")) with io.open(path, "wb") as f: f.write(data) proc = IOProcess(timeout=10, max_threads=5) with closing(proc): read = proc.readfile(path) assert read == data
def testChmodFail(self): proc = IOProcess(timeout=10, max_threads=5) with closing(proc): try: proc.chmod("/I/do/not/exist", 0) except OSError as e: if e.errno != errno.ENOENT: raise else: raise AssertionError("OSError was not raised")
def testStatFail(self): proc = IOProcess(timeout=1, max_threads=5) with closing(proc): try: proc.stat("/I do not exist") except OSError as e: if e.errno != errno.ENOENT: raise else: raise AssertionError("OSError was not raised")
def testSymlinkFail(self): proc = IOProcess(timeout=10, max_threads=5) with closing(proc): try: proc.symlink("/Dsadsad", "/I/do/not/exist") except OSError as e: if e.errno != errno.ENOENT: raise else: raise AssertionError("OSError was not raised")
def test_access_file(tmpdir, mode, permission, expected_result): proc = IOProcess(timeout=10, max_threads=5) f = tmpdir.join("file") f.write("") path = str(f) with closing(proc): with chmod(path, mode): assert proc.access(path, permission) == os.access(path, permission) assert proc.access(path, permission) == expected_result
def testSymlinkFail(self): proc = IOProcess(timeout=1, max_threads=5) with closing(proc): try: proc.symlink("/Dsadsad", "/I/do/not/exist") except OSError as e: if e.errno != errno.ENOENT: raise else: raise AssertionError("OSError was not raised")
def testChmodFail(self): proc = IOProcess(timeout=1, max_threads=5) with closing(proc): try: proc.chmod("/I/do/not/exist", 0) except OSError as e: if e.errno != errno.ENOENT: raise else: raise AssertionError("OSError was not raised")
def testStatFail(self): proc = IOProcess(timeout=10, max_threads=5) with closing(proc): try: proc.stat("/I do not exist") except OSError as e: if e.errno != errno.ENOENT: raise else: raise AssertionError("OSError was not raised")
def testRecoverAfterCrash(self): data = """Brigadier: Is there anything I can do? Third Doctor: Yes, pass me a silicon rod. [Stirs cup of tea with it] Brigadier: I meant is there anything UNIT can do about this space lightning business?""" # (C) BBC - Doctor Who proc = IOProcess(timeout=1, max_threads=5) with closing(proc): self.assertEquals(proc.echo(data), data) self.assertTrue(proc.crash()) self.assertEquals(proc.echo(data), data)
def test_stat_link_missing(tmpdir): proc = IOProcess(timeout=10, max_threads=5) with closing(proc): src = str(tmpdir.join("file")) link = str(tmpdir.join("link")) os.symlink(src, link) with pytest.raises(OSError) as myerror: proc.stat(link) with pytest.raises(OSError) as pyerror: os.stat(link) assert myerror.value.errno == pyerror.value.errno
def test_stat_link_missing(tmpdir): proc = IOProcess(timeout=1, max_threads=5) with closing(proc): src = str(tmpdir.join("file")) link = str(tmpdir.join("link")) os.symlink(src, link) with pytest.raises(OSError) as myerror: proc.stat(link) with pytest.raises(OSError) as pyerror: os.stat(link) assert myerror.value.errno == pyerror.value.errno
def testRecoverAfterCrash(self): data = """Brigadier: Is there anything I can do? Third Doctor: Yes, pass me a silicon rod. [Stirs cup of tea with it] Brigadier: I meant is there anything UNIT can do about this space lightning business?""" # (C) BBC - Doctor Who proc = IOProcess(timeout=10, max_threads=5) with closing(proc): self.assertEquals(proc.echo(data), data) self.assertTrue(proc.crash()) self.assertEquals(proc.echo(data), data)
def test_close_unrelated_fds(self): # Make inheritable file descriptor. with open(__file__) as my_file: clear_cloexec(my_file.fileno()) proc = IOProcess(timeout=10, max_threads=5) with closing(proc): # Wait until ready. proc.ping() proc_fd = "/proc/%d/fd" % proc.pid child_fds = [int(fd) for fd in os.listdir(proc_fd)] # My file descriptor must not be inherited. self.assertNotIn(my_file.fileno(), child_fds)
def testUnlink(self): proc = IOProcess(timeout=1, max_threads=5) with closing(proc): fd, path = mkstemp() try: os.close(fd) self.assertTrue(proc.unlink(path)) finally: try: os.unlink(path) except: pass
def test_close_unrelated_fds(self): # Make inheritable file descriptor. with open(__file__) as my_file: clear_cloexec(my_file.fileno()) proc = IOProcess(timeout=1, max_threads=5) with closing(proc): # Wait until ready. proc.ping() proc_fd = "/proc/%d/fd" % proc.pid child_fds = [int(fd) for fd in os.listdir(proc_fd)] # My file descriptor must not be inherited. self.assertNotIn(my_file.fileno(), child_fds)
def testUnlink(self): proc = IOProcess(timeout=1, max_threads=5) with closing(proc): fd, path = mkstemp() try: os.close(fd) self.assertTrue(proc.unlink(path)) finally: try: os.unlink(path) except: pass
def testLexists(self): proc = IOProcess(timeout=10, max_threads=5) with closing(proc): path = "/tmp/linktest.ioprocesstest" try: os.unlink(path) except OSError: pass os.symlink("dsadsadsadsad", path) try: self.assertTrue(proc.lexists(path)) finally: os.unlink(path)
def testRmdir(self): proc = IOProcess(timeout=10, max_threads=5) with closing(proc): path = mkdtemp() try: proc.rmdir(path) self.assertFalse(os.path.exists(path)) finally: try: shutil.rmtree(path) except: pass
def testTimeout(self): data = """Madge: Are you the new caretaker? The Doctor: Usually called "The Doctor." Or "The Caretaker." Or "Get off this planet." Though, strictly speaking, that probably isn't a name.""" # (C) BBC - Doctor Who proc = IOProcess(timeout=1, max_threads=5) with closing(proc): try: self.assertEquals(proc.echo(data, 10), data) except Timeout: return self.fail("Exception not raised")
def testChmod(self): proc = IOProcess(timeout=10, max_threads=5) with closing(proc): fd, path = mkstemp() targetMode = os.W_OK | os.R_OK try: os.chmod(path, 0) os.close(fd) self.assertFalse(os.stat(path).st_mode & targetMode) self.assertTrue(proc.chmod(path, targetMode)) self.assertTrue(os.stat(path).st_mode & targetMode) finally: os.unlink(path)
def testChmod(self): proc = IOProcess(timeout=1, max_threads=5) with closing(proc): fd, path = mkstemp() targetMode = os.W_OK | os.R_OK try: os.chmod(path, 0) os.close(fd) self.assertFalse(os.stat(path).st_mode & targetMode) self.assertTrue(proc.chmod(path, targetMode)) self.assertTrue(os.stat(path).st_mode & targetMode) finally: os.unlink(path)
def testRmdir(self): proc = IOProcess(timeout=1, max_threads=5) with closing(proc): path = mkdtemp() try: proc.rmdir(path) self.assertFalse(os.path.exists(path)) finally: try: shutil.rmtree(path) except: pass
def testLexists(self): proc = IOProcess(timeout=1, max_threads=5) with closing(proc): path = "/tmp/linktest.ioprocesstest" try: os.unlink(path) except OSError: pass os.symlink("dsadsadsadsad", path) try: self.assertTrue(proc.lexists(path)) finally: os.unlink(path)
def test_probe_block_size_not_writable(tmpdir): no_write = str(tmpdir.mkdir("no-write-for-you")) # Remove write bit, so the probe file cannot be created. with chmod(no_write, 0o500): proc = IOProcess(timeout=10, max_threads=5) proc._sublog = FakeLogger() with closing(proc): with pytest.raises(OSError) as e: proc.probe_block_size(no_write) assert e.value.errno == errno.EACCES warning = re.compile(r"Failed to create a probe file: .+?, " r"error: 'Permission denied'") assert any(warning.search(msg) for msg in proc._sublog.messages)
def testTimeout(self): data = """Madge: Are you the new caretaker? The Doctor: Usually called "The Doctor." Or "The Caretaker." Or "Get off this planet." Though, strictly speaking, that probably isn't a name.""" # (C) BBC - Doctor Who proc = IOProcess(timeout=1, max_threads=5) with closing(proc): try: self.assertEquals(proc.echo(data, 10), data) except Timeout: return self.fail("Exception not raised")
def testLink(self): proc = IOProcess(timeout=1, max_threads=5) with closing(proc): fd, oldpath = mkstemp() newpath = oldpath + ".new" try: os.close(fd) self.assertTrue(proc.link(oldpath, newpath)) finally: os.unlink(oldpath) try: os.unlink(newpath) except: pass
def testSymlink(self): proc = IOProcess(timeout=1, max_threads=5) with closing(proc): fd, oldpath = mkstemp() newpath = oldpath + ".new" try: os.close(fd) self.assertTrue(proc.symlink(oldpath, newpath)) self.assertEquals(os.path.realpath(newpath), os.path.normpath(oldpath)) finally: os.unlink(oldpath) try: os.unlink(newpath) except: pass
def testGlob(self): proc = IOProcess(timeout=1, max_threads=5) with closing(proc): path = mkdtemp() matches = [] for i in range(10): matches.append(os.path.join(path, str(i))) with open(matches[-1], "w") as f: f.write("A") matches.sort() try: remoteMatches = proc.glob(os.path.join(path, "*")) remoteMatches.sort() self.assertEquals(remoteMatches, matches) finally: shutil.rmtree(path)
def testReadfile(self, direct=False): data = b'''The Doctor: Well... you could do that. Yeah, you could do that. Of course you could! But why? Look at these people, these human beings. Consider their potential! From the day they arrive on the planet, blinking, step into the sun, there is more to see than can ever be seen, more to do than-no, hold on. Sorry, that's The Lion King. But the point still stands: leave them alone! ''' # (C) BBC - Doctor Who proc = IOProcess(timeout=1, max_threads=5) with closing(proc): fd, path = mkstemp(dir="/var/tmp") try: os.write(fd, data) os.close(fd) remoteData = proc.readfile(path, direct) self.assertEquals(remoteData[:len(data)], data) finally: os.unlink(path)
def testWritefile(self, direct=False): data = b'''Jackie: I'm in my dressing gown. The Doctor: Yes, you are. Jackie: There's a strange man in my bedroom. The Doctor: Yes, there is. Jackie: Anything could happen. The Doctor: No. [walks away]''' # (C) BBC - Doctor Who # This test sometimes time out in the CI. On one failure, the write # took 1.8 seconds inside ioprocess. proc = IOProcess(timeout=5, max_threads=5) with closing(proc): fd, path = mkstemp(dir="/var/tmp") try: os.close(fd) proc.writefile(path, data, direct) with open(path, 'rb') as f: diskData = f.read() self.assertEquals(diskData[:len(data)], data) finally: os.unlink(path)
def testMaxRequests(self): proc = IOProcess(timeout=5, max_threads=1, max_queued_requests=1) with closing(proc): t1 = Thread(target=proc.echo, args=("hello", 2)) t2 = Thread(target=proc.echo, args=("hello", 2)) t1.start() t2.start() # Make sure the echo calls are sent prior to the ping otherwise one # of them would fail and ping() would pass time.sleep(0.5) try: proc.ping() except OSError as e: self.assertEquals(e.errno, errno.EAGAIN) except Exception: self.fail("Expected OSError got %s", e) else: self.fail("Expected exception") finally: t1.join() t2.join()
def testMaxRequestsAfterFillingThreadPool(self): proc = IOProcess(timeout=5, max_threads=3, max_queued_requests=0) with closing(proc): t1 = Thread(target=proc.echo, args=("hello", 2)) t2 = Thread(target=proc.echo, args=("hello", 2)) t3 = Thread(target=proc.echo, args=("hello", 2)) t1.start() t2.start() t3.start() for t in (t1, t2, t3): t.join() t1 = Thread(target=proc.echo, args=("hello", 2)) t2 = Thread(target=proc.echo, args=("hello", 2)) t1.start() t2.start() # Make sure the echo calls are sent prior to the ping otherwise one # of them would fail and ping() would pass time.sleep(0.5) proc.ping() t1.join() t2.join()
def test_partial_logs(self): threads = [] proc = IOProcess(timeout=1, max_threads=10) proc._sublog = FakeLogger() def worker(): for i in range(100): proc.stat(__file__) try: for i in range(4): t = Thread(target=worker) t.deamon = True t.start() threads.append(t) finally: for t in threads: t.join() proc.close() for msg in proc._sublog.messages: self.assertFalse('DEBUG|' in msg, "Raw log data in log message: %r" % msg)
def testPendingRequestInvalidationOnCrash(self): data = """The Doctor: A straight line may be the shortest distance between two points, but it is by no means the most interesting.""" # (C) BBC - Doctor Who proc = IOProcess(timeout=12, max_threads=5) with closing(proc): res = [False] def sendCmd(): try: proc.echo(data, 10) except OSError as e: if e.errno == ERR_IOPROCESS_CRASH: res[0] = True else: log.error("Got unexpected error", exc_info=True) t = Thread(target=sendCmd) t.start() time.sleep(1) proc.crash() t.join() self.assertTrue(res[0])
def testManyRequests(self): data = """Lily: What's happening? The Doctor: No idea. Just do what I do: hold tight and pretend it's a plan.""" # (C) BBC - Doctor Who proc = IOProcess(timeout=30, max_threads=5) with closing(proc): # even though we theoretically go back to a stable state, some # objects might have increased their internal buffers and mem # fragmantation might have caused some data to be spanned on more # pages then it originally did. acceptableRSSIncreasKB = 100 startRSS = proc.memstat()['rss'] # This way we catch evey leak that is more then one 0.1KB per call many = 300 for i in range(many): self.assertEquals(proc.echo(data), data) endRSS = proc.memstat()['rss'] RSSDiff = endRSS - startRSS log.debug("RSS difference was %d KB, %d per request", RSSDiff, RSSDiff / many) # This only tests for leaks in the main request\response process. self.assertTrue(RSSDiff < acceptableRSSIncreasKB, "Detected a leak sized %d KB" % RSSDiff)