def testTrivialPoll(self):
nopen = self.numOpenFiles()
pl = Pipeline(("sleep", "1"))
while not pl.poll():
pass
pl.wait()
self.commonChecks(nopen, pl, "sleep 1 2>[DataReader]")
def testBogusStderr(self):
# test stderr specification is not legal
nopen = self.numOpenFiles()
with self.assertRaisesRegex(PipettorException, self.__bogusStdioExpectRe()):
pl = Pipeline([("date",), ("date",)], stderr=3.14159)
pl.wait()
self.orphanChecks(nopen)
def testPipeFailStderr(self):
nopen = self.numOpenFiles()
# should report first failure
pl = Pipeline([("true",), (os.path.join(self.getTestDir(), "progWithError"),), ("false",)], stderr=DataReader)
with self.assertRaises(ProcessException) as cm:
pl.wait()
self.checkProgWithError(cm.exception)
self.orphanChecks(nopen)
def testDataReaderBogusShare(self):
# test stderr specification is not legal
nopen = self.numOpenFiles()
dr = DataReader()
with self.assertRaisesRegex(PipettorException, "^DataReader already bound to a process$"):
pl = Pipeline([("date",), ("date",)], stdout=dr, stderr=dr)
pl.wait()
self.orphanChecks(nopen)
def testIntArg(self):
nopen = self.numOpenFiles()
inf = self.getInputFile("simple1.txt")
dr = DataReader()
pl = Pipeline(("head", -2), stdin=inf, stdout=dr)
pl.wait()
self.assertEqual(dr.data, "one\ntwo\n")
self.commonChecks(nopen, pl, "^head -2 <.+/input/simple1\\.txt >\\[DataReader\\]", isRe=True)
def testSimplePipe(self):
nopen = self.numOpenFiles()
log = LoggerForTests()
pl = Pipeline([("true",), ("true",)], logger=log.logger)
pl.wait()
self.commonChecks(nopen, pl, "true | true 2>[DataReader]")
self.assertEqual(log.data, """start: true | true 2>[DataReader]\n"""
"""success: true | true 2>[DataReader]\n""")
def testReadFile(self):
# test read and write to File object
nopen = self.numOpenFiles()
inf = self.getInputFile("simple1.txt")
outf = self.getOutputFile(".out")
pl = Pipeline([("cat",), ("cat",)], stdin=File(inf), stdout=File(outf, "w"))
pl.wait()
self.diffExpected(".out")
self.commonChecks(nopen, pl, "cat <.*/input/simple1.txt \\| cat >.*/output/test_pipettor.PipelineTests.testReadFile.out 2>\\[DataReader\\]$", isRe=True)
def testStdinStdoutMem(self):
# write and read from memory
nopen = self.numOpenFiles()
dw = DataWriter("one\ntwo\nthree\n")
dr = DataReader()
pl = Pipeline([("cat", "-u"), ("cat", "-u")], stdin=dw, stdout=dr)
pl.wait()
self.assertEqual(dr.data, "one\ntwo\nthree\n")
self.commonChecks(nopen, pl, "^cat -u <\\[DataWriter\\] \\| cat -u >\\[DataReader\\] 2>\\[DataReader\\]$", isRe=True)
def testStdoutMem(self):
# read from stdout into memory
nopen = self.numOpenFiles()
inf = self.getInputFile("simple1.txt")
dr = DataReader()
pl = Pipeline(("sort", "-r"), stdin=inf, stdout=dr)
pl.wait()
self.assertEqual(dr.data, "two\nthree\nsix\none\nfour\nfive\n")
self.commonChecks(nopen, pl, "^sort -r <.+/input/simple1\\.txt >\\[DataReader\\]", isRe=True)
def testStdinMem(self):
# write from memory to stdin
nopen = self.numOpenFiles()
outf = self.getOutputFile(".out")
dw = DataWriter("one\ntwo\nthree\n")
pl = Pipeline(("sort", "-r"), stdin=dw, stdout=outf)
pl.wait()
self.diffExpected(".out")
self.commonChecks(nopen, pl, "^sort -r <\\[DataWriter\\] >.+/output/test_pipettor\\.PipelineTests\\.testStdinMem\\.out 2>\\[DataReader\\]$", isRe=True)
def testDataWriterBogusShare(self):
# test stderr specification is not legal
nopen = self.numOpenFiles()
dw = DataWriter("fred")
with self.assertRaisesRegex(PipettorException, "^DataWriter already bound to a process$"):
pl1 = Pipeline([("cat", "/dev/null"), ("cat", "/dev/null")], stdin=dw)
Pipeline([("cat", "/dev/null"), ("cat", "/dev/null")], stdin=dw)
pl1.shutdown() # clean up unstarted process
self.orphanChecks(nopen)
def testStdinMemBinary(self):
# binary write from memory to stdin
nopen = self.numOpenFiles()
outf = self.getOutputFile(".out")
fh = open(self.getInputFile("file.binary"), "rb")
dw = DataWriter(fh.read())
fh.close()
pl = Pipeline(("cat",), stdin=dw, stdout=outf)
pl.wait()
self.diffBinaryExpected(".out", expectedBasename="file.binary")
self.commonChecks(nopen, pl, "^cat <\\[DataWriter] >.*/output/test_pipettor.PipelineTests.testStdinMemBinary.out 2>\\[DataReader\\]$", isRe=True)
def testCollectStdoutErr(self):
# independent collection of stdout and stderr
nopen = self.numOpenFiles()
stdoutRd = DataReader()
stderrRd = DataReader()
pl = Pipeline(("sh", "-c", "echo this goes to stdout; echo this goes to stderr >&2"),
stdout=stdoutRd, stderr=stderrRd)
pl.wait()
self.assertEqual(stdoutRd.data, "this goes to stdout\n")
self.assertEqual(stderrRd.data, "this goes to stderr\n")
self.commonChecks(nopen, pl, "sh -c 'echo this goes to stdout; echo this goes to stderr >&2' >[DataReader] 2>[DataReader]")
def testSimplePipeFail(self):
nopen = self.numOpenFiles()
log = LoggerForTests()
pl = Pipeline([("false",), ("true",)], logger=log.logger)
with self.assertRaisesRegex(ProcessException, "^process exited 1: false$"):
pl.wait()
self.commonChecks(nopen, pl, "false | true 2>[DataReader]")
self.assertRegex(log.data,
re.compile("""^start: false | true 2>[DataReader]\n"""
"""failure: false | true 2>[DataReader]: process exited 1: false\n.*""",
re.MULTILINE))
def testSignaled(self):
# process signals
nopen = self.numOpenFiles()
pl = Pipeline(("sh", "-c", "kill -11 $$"))
with self.assertRaises(ProcessException) as cm:
pl.wait()
expect = "process signaled: SIGSEGV: sh -c 'kill -11 $$'"
msg = str(cm.exception)
if not msg.startswith(expect):
self.fail("'" + msg + "' does not start with '"
+ expect + "', cause: " + str(getattr(cm.exception, "cause", None)))
self.commonChecks(nopen, pl, "sh -c 'kill -11 $$' 2>[DataReader]")
def testStdoutMemBinary(self):
# binary read from stdout into memory
nopen = self.numOpenFiles()
inf = self.getInputFile("file.binary")
dr = DataReader(binary=True)
pl = Pipeline(("cat",), stdin=inf, stdout=dr)
pl.wait()
fh = open(self.getOutputFile(".out"), "wb")
fh.write(dr.data)
fh.close()
self.diffBinaryExpected(".out", expectedBasename="file.binary")
self.commonChecks(nopen, pl, "^cat <.*/input/file.binary >\\[DataReader] 2>\\[DataReader\\]$", isRe=True)
def testExecFail(self):
# invalid executable
nopen = self.numOpenFiles()
dw = DataWriter("one\ntwo\nthree\n")
pl = Pipeline(("procDoesNotExist", "-r"), stdin=dw)
with self.assertRaises(ProcessException) as cm:
pl.wait()
self.assertRegex(str(cm.exception),
"exec failed: procDoesNotExist -r.*")
self.assertIsNot(cm.exception.__cause__, None)
self.assertRegex(str(cm.exception.__cause__),
"\\[Errno 2\\] No such file or directory: 'procDoesNotExist'.*")
self.commonChecks(nopen, pl, "procDoesNotExist -r <[DataWriter] 2>[DataReader]")
def testPipeFail3Stderr(self):
# all 3 process fail
nopen = self.numOpenFiles()
# should report first failure
pl = Pipeline([(os.path.join(self.getTestDir(), "progWithError"), "process0"),
(os.path.join(self.getTestDir(), "progWithError"), "process1"),
(os.path.join(self.getTestDir(), "progWithError"), "process2")],
stderr=DataReader)
with self.assertRaises(ProcessException) as cm:
pl.wait()
# should be first process
self.checkProgWithError(cm.exception, "process0")
# check process
for i in range(3):
self.checkProgWithError(pl.procs[i].procExcept, "process{}".format(i))
self.orphanChecks(nopen)
def testAppendFile(self):
# test append to File object
nopen = self.numOpenFiles()
inf = self.getInputFile("simple1.txt")
outf = self.getOutputFile(".out")
# double cat actually found a bug
pl = Pipeline([("cat",), ("cat",)], stdin=inf, stdout=File(outf, "w"), stderr=None)
pl.wait()
pl = Pipeline([("cat",), ("cat",)], stdin=inf, stdout=File(outf, "a"), stderr=None)
pl.wait()
self.diffExpected(".out")
self.commonChecks(nopen, pl, "cat <.*/input/simple1.txt \\| cat >.*/output/test_pipettor.PipelineTests.testAppendFile.out$", isRe=True)
def testRead(self):
nopen = self.numOpenFiles()
inf = self.getInputFile("simple1.txt")
infGz = self.getOutputFile(".txt.gz")
Pipeline(("gzip", "-c", inf), stdout=infGz).wait()
pl = Popen(("gzip", "-dc"), "r", stdin=infGz)
self.cpPlToFile(pl, ".out")
pl.wait()
self.diffExpected(".out")
self.commonChecks(nopen, pl, "^gzip -dc <.*output/test_pipettor.PopenTests.testRead.txt.gz >.+$", isRe=True)
def testWrite(self):
nopen = self.numOpenFiles()
outf = self.getOutputFile(".out")
outfGz = self.getOutputFile(".out.gz")
pl = Popen(("gzip", "-1"), "w", stdout=outfGz)
self.cpFileToPl("simple1.txt", pl)
pl.close()
self.commonChecks(nopen, pl, "gzip -1 <.+ >.*output/test_pipettor.PopenTests.testWrite.out.gz", isRe=True)
Pipeline(("zcat", outfGz), stdout=outf).wait()
self.diffExpected(".out")
def testTrivialStatus(self):
nopen = self.numOpenFiles()
pl = Pipeline(("true",))
pl.start()
self.assertTrue(pl.running)
self.assertFalse(pl.finished)
pl.wait()
self.assertFalse(pl.running)
self.assertTrue(pl.finished)
self.commonChecks(nopen, pl, "true 2>[DataReader]")
def testTrivialFailPoll(self):
nopen = self.numOpenFiles()
pl = Pipeline([("sleep", "1"), ("false",)])
with self.assertRaisesRegex(ProcessException, "^process exited 1: sleep 1 | false$"):
pl.wait()
self.commonChecks(nopen, pl, "sleep 1 | false 2>[DataReader]")
def testTrivial(self):
nopen = self.numOpenFiles()
pl = Pipeline(("true",))
pl.wait()
self.commonChecks(nopen, pl, "true 2>[DataReader]")