def testJoinPipe2Uniq(self): "cat from two pipeline of two sorts each, results pipe to sort -u" nopen = self.numOpenFiles() pd = ProcDag() inf = self.getInputFile("simple1.txt") outf = self.getOutputFile(".out") fwd1 = Pipe() fwd2 = Pipe() rev1 = Pipe() rev2 = Pipe() sortu = Pipe() # sort reverse direction, then sort again into final orders pd.create(("sort", "-r"), stdin=inf, stdout=fwd1) pd.create(("sort", ), stdin=fwd1, stdout=fwd2) pd.create(("sort", ), stdin=inf, stdout=rev1) pd.create(("sort", "-r"), stdin=rev1, stdout=rev2) pd.create(("cat", PIn(fwd2), PIn(rev2)), stdout=sortu) pd.create(("sort", "-u"), stdin=sortu, stdout=outf) pd.wait() self.diffExpected(".out") self.commonChecks( nopen, pd, "^cat <\\(sort -r <.+/tests/input/simple1.txt \\| sort\\) <\\(sort <.+/input/simple1.txt \\| sort -r\\) \\| sort -u >.+/output/ProcDagTests\\.ProcDagTests\\.testJoinPipe2Uniq\\.out$", isRe=True)
def testSplitPipe2(self): "tee into two pipeline of two sorts each " nopen = self.numOpenFiles() pd = ProcDag() inf = self.getInputFile("simple1.txt") outFwd = self.getOutputFile(".fwd.out") outRev = self.getOutputFile(".rev.out") fwd1 = Pipe() fwd2 = Pipe() rev1 = Pipe() rev2 = Pipe() # sort wrong direction, then sort again into final orders pd.create(("tee", POut(rev1)), stdin=inf, stdout=POut(fwd1)) pd.create(("sort", "-r"), stdin=fwd1, stdout=fwd2) pd.create(("sort", ), stdin=fwd2, stdout=outFwd) pd.create(("sort", ), stdin=rev1, stdout=rev2) pd.create(("sort", "-r"), stdin=rev2, stdout=outRev) pd.wait() self.diffExpected(".fwd.out") self.diffExpected(".rev.out") self.commonChecks( nopen, pd, "^tee >\\(sort \\| sort -r >.+/output/ProcDagTests\\.ProcDagTests\\.testSplitPipe2\\.rev\\.out\\) <.+/input/simple1\\.txt \\| sort -r \\| sort >.+/output/ProcDagTests\\.ProcDagTests\\.testSplitPipe2\\.fwd\\.out$", isRe=True)
def testSimplePipe(self): nopen = self.numOpenFiles() pd = ProcDag() p = Pipe() pd.create(("true", ), stdout=p) pd.create(("true", ), stdin=p) pd.wait() self.commonChecks(nopen, pd, "true | true")
def testOutArgNoOpen(self): "write to a pipe argument that doesn't get open" nopen = self.numOpenFiles() pd = ProcDag() dr = DataReader() pr = pd.create(("true", POut(dr)), stdin="/dev/null") pd.wait() self.commonChecks(nopen, pd, "true [DataWriter] </dev/null")
def testInArgNoOpen(self): "read from a pipe argument that doesn't get open" nopen = self.numOpenFiles() pd = ProcDag() dw = DataWriter("one\ntwo\nthree\n") pr = pd.create(("true", PIn(dw)), stdin="/dev/null") pd.wait() self.commonChecks(nopen, pd, "true [DataWriter] </dev/null")
def testInArgNoOpen(self): "read from a pipe argument that doesn't get open" nopen = self.numOpenFiles() pd = ProcDag() dw = DataWriter("one\ntwo\nthree\n") pr = pd.create(("true", PIn(dw)), stdin="/dev/null") pd.wait() self.commonChecks(nopen, pd, "true [DataWriter] </dev/null")
def testOutArgNoOpen(self): "write to a pipe argument that doesn't get open" nopen = self.numOpenFiles() pd = ProcDag() dr = DataReader() pr = pd.create(("true", POut(dr)), stdin="/dev/null") pd.wait() self.commonChecks(nopen, pd, "true [DataWriter] </dev/null")
def testSimplePipe(self): nopen = self.numOpenFiles() pd = ProcDag() p = Pipe() pd.create(("true",), stdout=p) pd.create(("true",), stdin=p) pd.wait() self.commonChecks(nopen, pd, "true | true")
def testInArgMem(self): "write from memory to a pipe argument" nopen = self.numOpenFiles() pd = ProcDag() dw = DataWriter("one\ntwo\nthree\n") outf = self.getOutputFile(".out") pr = pd.create(("sort","-r", PIn(dw)), stdin="/dev/null", stdout=outf) pd.wait() self.diffExpected(".out") self.commonChecks(nopen, pd, "^sort -r \\[DataWriter\\] </dev/null >.+/output/ProcDagTests\\.ProcDagTests\\.testInArgMem\\.out$", isRe=True)
def testStdoutMem(self): "read from stdout into memory" nopen = self.numOpenFiles() pd = ProcDag() inf = self.getInputFile("simple1.txt") dr = DataReader() pd.create(("sort","-r"), stdin=inf, stdout=dr) pd.wait() self.failUnlessEqual(dr.get(), "two\nthree\nsix\none\nfour\nfive\n") self.commonChecks(nopen, pd, "^sort -r <.+/input/simple1\\.txt >\\[DataWriter\\]", isRe=True)
def testStdinMem(self): "write from memory to stdin" nopen = self.numOpenFiles() pd = ProcDag() dw = DataWriter("one\ntwo\nthree\n") outf = self.getOutputFile(".out") pd.create(("sort","-r"), stdin=dw, stdout=outf) pd.wait() self.diffExpected(".out") self.commonChecks(nopen, pd, "^sort -r <\\[DataWriter\\] >.+/output/ProcDagTests\\.ProcDagTests\\.testStdinMem\\.out$", isRe=True)
def testOutArgMem(self): "read from a pipe argument into memory" nopen = self.numOpenFiles() pd = ProcDag() inf = self.getInputFile("simple1.txt") dr = DataReader() pr = pd.create(("tee",POut(dr)), stdin=inf, stdout="/dev/null") pd.wait() self.failUnlessEqual(dr.get(), "one\ntwo\nthree\nfour\nfive\nsix\n") self.commonChecks(nopen, pd, "^tee \\[DataWriter\\] <.+/input/simple1.txt >/dev/null$", isRe=True)
def testTrivialFail(self): nopen = self.numOpenFiles() pd = ProcDag() pd.create(("false",)) err = None try: pd.wait() except Exception as err: pass self.failUnless(isinstance(err, ProcException)) self.commonChecks(nopen, pd, "false")
def testTrivialFail(self): nopen = self.numOpenFiles() pd = ProcDag() pd.create(("false", )) err = None try: pd.wait() except Exception as err: pass self.failUnless(isinstance(err, ProcException)) self.commonChecks(nopen, pd, "false")
def testStdoutToArg(self): "stdout to a pipe argument" nopen = self.numOpenFiles() pd = ProcDag() inf = self.getInputFile("simple1.txt") outf = self.getOutputFile(".out") io = Pipe() pd.create(("cat", inf), stdout=io) pd.create(("sort","-r", PIn(io)), stdin="/dev/null", stdout=outf) pd.wait() self.diffExpected(".out") self.commonChecks(nopen, pd, "^sort -r <\\(cat .+/input/simple1.txt\\) </dev/null >.+/output/ProcDagTests\\.ProcDagTests\\.testStdoutToArg\\.out$", isRe=True)
def testArgToStdout(self): "pipe argument to stdout" nopen = self.numOpenFiles() pd = ProcDag() inf = self.getInputFile("simple1.txt") outf = self.getOutputFile(".out") io = Pipe() pd.create(("tee", POut(io)), stdin=inf, stdout="/dev/null") pd.create(("sort","-r"), stdin=io, stdout=outf) pd.wait() self.diffExpected(".out") self.commonChecks(nopen, pd, "^tee >\\(sort -r >.+/output/ProcDagTests\\.ProcDagTests\\.testArgToStdout\\.out\\) <.+/input/simple1\\.txt >/dev/null$", isRe=True)
def testSimplePipeFail(self): nopen = self.numOpenFiles() pd = ProcDag() p = Pipe() pd.create(("false", ), stdout=p) pd.create(("true", ), stdin=p) err = None try: pd.wait() except Exception as err: pass self.failUnless(isinstance(err, ProcException)) self.commonChecks(nopen, pd, "false | true")
def testStdoutMem(self): "read from stdout into memory" nopen = self.numOpenFiles() pd = ProcDag() inf = self.getInputFile("simple1.txt") dr = DataReader() pd.create(("sort", "-r"), stdin=inf, stdout=dr) pd.wait() self.failUnlessEqual(dr.get(), "two\nthree\nsix\none\nfour\nfive\n") self.commonChecks(nopen, pd, "^sort -r <.+/input/simple1\\.txt >\\[DataWriter\\]", isRe=True)
def testSimplePipeFail(self): nopen = self.numOpenFiles() pd = ProcDag() p = Pipe() pd.create(("false",), stdout=p) pd.create(("true",), stdin=p) err = None try: pd.wait() except Exception as err: pass self.failUnless(isinstance(err, ProcException)) self.commonChecks(nopen, pd, "false | true")
def testOutArgMem(self): "read from a pipe argument into memory" nopen = self.numOpenFiles() pd = ProcDag() inf = self.getInputFile("simple1.txt") dr = DataReader() pr = pd.create(("tee", POut(dr)), stdin=inf, stdout="/dev/null") pd.wait() self.failUnlessEqual(dr.get(), "one\ntwo\nthree\nfour\nfive\nsix\n") self.commonChecks( nopen, pd, "^tee \\[DataWriter\\] <.+/input/simple1.txt >/dev/null$", isRe=True)
def testStdinMem(self): "write from memory to stdin" nopen = self.numOpenFiles() pd = ProcDag() dw = DataWriter("one\ntwo\nthree\n") outf = self.getOutputFile(".out") pd.create(("sort", "-r"), stdin=dw, stdout=outf) pd.wait() self.diffExpected(".out") self.commonChecks( nopen, pd, "^sort -r <\\[DataWriter\\] >.+/output/ProcDagTests\\.ProcDagTests\\.testStdinMem\\.out$", isRe=True)
def testInArgMem(self): "write from memory to a pipe argument" nopen = self.numOpenFiles() pd = ProcDag() dw = DataWriter("one\ntwo\nthree\n") outf = self.getOutputFile(".out") pr = pd.create(("sort", "-r", PIn(dw)), stdin="/dev/null", stdout=outf) pd.wait() self.diffExpected(".out") self.commonChecks( nopen, pd, "^sort -r \\[DataWriter\\] </dev/null >.+/output/ProcDagTests\\.ProcDagTests\\.testInArgMem\\.out$", isRe=True)
def testSplitPipe(self): "tee into two sorts" nopen = self.numOpenFiles() pd = ProcDag() inf = self.getInputFile("simple1.txt") outFwd = self.getOutputFile(".fwd.out") outRev = self.getOutputFile(".rev.out") fwd = Pipe() rev = Pipe() pd.create(("tee", POut(rev)), stdin=inf, stdout=POut(fwd)) pd.create(("sort",), stdin=fwd, stdout=outFwd) pd.create(("sort","-r"), stdin=rev, stdout=outRev) pd.wait() self.diffExpected(".fwd.out") self.diffExpected(".rev.out") self.commonChecks(nopen, pd, "^tee >\\(sort -r >.+/output/ProcDagTests\\.ProcDagTests\\.testSplitPipe\\.rev\\.out\\) <.+/input/simple1\\.txt \\| sort >.+/output/ProcDagTests\\.ProcDagTests\\.testSplitPipe\\.fwd\\.out$", isRe=True)
def testStdoutToArg(self): "stdout to a pipe argument" nopen = self.numOpenFiles() pd = ProcDag() inf = self.getInputFile("simple1.txt") outf = self.getOutputFile(".out") io = Pipe() pd.create(("cat", inf), stdout=io) pd.create(("sort", "-r", PIn(io)), stdin="/dev/null", stdout=outf) pd.wait() self.diffExpected(".out") self.commonChecks( nopen, pd, "^sort -r <\\(cat .+/input/simple1.txt\\) </dev/null >.+/output/ProcDagTests\\.ProcDagTests\\.testStdoutToArg\\.out$", isRe=True)
def testArgToStdout(self): "pipe argument to stdout" nopen = self.numOpenFiles() pd = ProcDag() inf = self.getInputFile("simple1.txt") outf = self.getOutputFile(".out") io = Pipe() pd.create(("tee", POut(io)), stdin=inf, stdout="/dev/null") pd.create(("sort", "-r"), stdin=io, stdout=outf) pd.wait() self.diffExpected(".out") self.commonChecks( nopen, pd, "^tee >\\(sort -r >.+/output/ProcDagTests\\.ProcDagTests\\.testArgToStdout\\.out\\) <.+/input/simple1\\.txt >/dev/null$", isRe=True)
def testArgToArg(self): "pipe argument to argument of another process" nopen = self.numOpenFiles() pd = ProcDag() inf = self.getInputFile("simple1.txt") outf = self.getOutputFile(".out") io = Pipe() pd.create(("tee", POut(io)), stdin=inf, stdout="/dev/null") pd.create(("cat", PIn(io)), stdin="/dev/null", stdout=outf) pd.wait() self.diffExpected(".out") # formatting fails here: (FIXME: improve??), FIXME: doesn't always produce same result, # so we check multiple posibilities self.commonChecks(nopen, pd, "(^tee >\\(cat <\\(tee \\.\\.\\.\\) </dev/null >.+/output/ProcDagTests\\.ProcDagTests\\.testArgToArg\\.out\\) <.+/input/simple1\\.txt >/dev/null ; cat \\.\\.\\.$)" \ + "|(^tee >\\(cat <\\(tee \\.\\.\\.\\) </dev/null >.+/output/ProcDagTests\\.ProcDagTests\\.testArgToArg\\.out\\) <.+/input/simple1\\.txt >/dev/null ; cat \\.\\.\\.$)", isRe=True)
def testExecFail(self): "invalid executable" nopen = self.numOpenFiles() pd = ProcDag() dw = DataWriter("one\ntwo\nthree\n") pd.create(("procDoesNotExist", "-r"), stdin=dw) ex = None try: pd.wait() except Exception as ex: pass expect = "exec failed: procDoesNotExist -r,\n caused by: OSError: [Errno 2] No such file or directory" msg = str(ex) if not msg.startswith(expect): self.fail("'" + msg + "' does not start with '" + expect + "', cause: " + str(getattr(ex, "cause", None))) self.commonChecks(nopen, pd, "procDoesNotExist -r <[DataWriter]")
def testArgToArg(self): "pipe argument to argument of another process" nopen = self.numOpenFiles() pd = ProcDag() inf = self.getInputFile("simple1.txt") outf = self.getOutputFile(".out") io = Pipe() pd.create(("tee", POut(io)), stdin=inf, stdout="/dev/null") pd.create(("cat", PIn(io)), stdin="/dev/null", stdout=outf) pd.wait() self.diffExpected(".out") # formatting fails here: (FIXME: improve??), FIXME: doesn't always produce same result, # so we check multiple posibilities self.commonChecks(nopen, pd, "(^tee >\\(cat <\\(tee \\.\\.\\.\\) </dev/null >.+/output/ProcDagTests\\.ProcDagTests\\.testArgToArg\\.out\\) <.+/input/simple1\\.txt >/dev/null ; cat \\.\\.\\.$)" \ + "|(^tee >\\(cat <\\(tee \\.\\.\\.\\) </dev/null >.+/output/ProcDagTests\\.ProcDagTests\\.testArgToArg\\.out\\) <.+/input/simple1\\.txt >/dev/null ; cat \\.\\.\\.$)", isRe=True)
def testExecFail(self): "invalid executable" nopen = self.numOpenFiles() pd = ProcDag() dw = DataWriter("one\ntwo\nthree\n") pd.create(("procDoesNotExist","-r"), stdin=dw) ex = None try: pd.wait() except Exception as ex: pass expect = "exec failed: procDoesNotExist -r,\n caused by: OSError: [Errno 2] No such file or directory" msg = str(ex) if not msg.startswith(expect): self.fail("'"+ msg + "' does not start with '" + expect + "', cause: " + str(getattr(ex,"cause", None))) self.commonChecks(nopen, pd, "procDoesNotExist -r <[DataWriter]")
def testJoinPipe2(self): "cat from two pipeline of two sorts each " nopen = self.numOpenFiles() pd = ProcDag() inf = self.getInputFile("simple1.txt") outf = self.getOutputFile(".out") fwd1 = Pipe() fwd2 = Pipe() rev1 = Pipe() rev2 = Pipe() # sort reverse direction, then sort again into final orders pd.create(("sort","-r"), stdin=inf, stdout=fwd1) pd.create(("sort",), stdin=fwd1, stdout=fwd2) pd.create(("sort",), stdin=inf, stdout=rev1) pd.create(("sort","-r"), stdin=rev1, stdout=rev2) pd.create(("cat", PIn(fwd2), PIn(rev2)), stdout=outf) pd.wait() self.diffExpected(".out") self.commonChecks(nopen, pd, "^cat <\\(sort -r <.+/input/simple1\\.txt \\| sort\\) <\\(sort <.+/input/simple1.txt \\| sort -r\\) >.+/output/ProcDagTests\\.ProcDagTests\\.testJoinPipe2\\.out$", isRe=True)
def testSplitPipe(self): "tee into two sorts" nopen = self.numOpenFiles() pd = ProcDag() inf = self.getInputFile("simple1.txt") outFwd = self.getOutputFile(".fwd.out") outRev = self.getOutputFile(".rev.out") fwd = Pipe() rev = Pipe() pd.create(("tee", POut(rev)), stdin=inf, stdout=POut(fwd)) pd.create(("sort", ), stdin=fwd, stdout=outFwd) pd.create(("sort", "-r"), stdin=rev, stdout=outRev) pd.wait() self.diffExpected(".fwd.out") self.diffExpected(".rev.out") self.commonChecks( nopen, pd, "^tee >\\(sort -r >.+/output/ProcDagTests\\.ProcDagTests\\.testSplitPipe\\.rev\\.out\\) <.+/input/simple1\\.txt \\| sort >.+/output/ProcDagTests\\.ProcDagTests\\.testSplitPipe\\.fwd\\.out$", isRe=True)
def testSplitPipe2(self): "tee into two pipeline of two sorts each " nopen = self.numOpenFiles() pd = ProcDag() inf = self.getInputFile("simple1.txt") outFwd = self.getOutputFile(".fwd.out") outRev = self.getOutputFile(".rev.out") fwd1 = Pipe() fwd2 = Pipe() rev1 = Pipe() rev2 = Pipe() # sort wrong direction, then sort again into final orders pd.create(("tee", POut(rev1)), stdin=inf, stdout=POut(fwd1)) pd.create(("sort","-r"), stdin=fwd1, stdout=fwd2) pd.create(("sort",), stdin=fwd2, stdout=outFwd) pd.create(("sort",), stdin=rev1, stdout=rev2) pd.create(("sort","-r"), stdin=rev2, stdout=outRev) pd.wait() self.diffExpected(".fwd.out") self.diffExpected(".rev.out") self.commonChecks(nopen, pd, "^tee >\\(sort \\| sort -r >.+/output/ProcDagTests\\.ProcDagTests\\.testSplitPipe2\\.rev\\.out\\) <.+/input/simple1\\.txt \\| sort -r \\| sort >.+/output/ProcDagTests\\.ProcDagTests\\.testSplitPipe2\\.fwd\\.out$", isRe=True)
def testStdioCycleDetect(self): "stdio cycle detection" nopen = self.numOpenFiles() pd = ProcDag() inf = self.getInputFile("simple1.txt") p1 = Pipe() p2 = Pipe() p3 = Pipe() p4 = Pipe() pd.create(("cat",), stdin=inf, stdout=p1) pd.create(("cat", "/dev/stdin", PIn(p4)), stdin=p1, stdout=p3) pd.create(("cat",), stdin=p3, stdout=p4) ex = None try: pd.wait() except ProcDagException as ex: pass msg = str(ex) expect = "cycle detected: entering: cat /dev/stdin <([Pipe])" if msg != expect: self.fail("'"+ msg + "' != '"+ expect + "'") self.commonChecks(nopen, pd, "{CYCLE}: cat ; cat ; cat /dev/stdin <([Pipe])")
def testStdioCycleDetect(self): "stdio cycle detection" nopen = self.numOpenFiles() pd = ProcDag() inf = self.getInputFile("simple1.txt") p1 = Pipe() p2 = Pipe() p3 = Pipe() p4 = Pipe() pd.create(("cat", ), stdin=inf, stdout=p1) pd.create(("cat", "/dev/stdin", PIn(p4)), stdin=p1, stdout=p3) pd.create(("cat", ), stdin=p3, stdout=p4) ex = None try: pd.wait() except ProcDagException as ex: pass msg = str(ex) expect = "cycle detected: entering: cat /dev/stdin <([Pipe])" if msg != expect: self.fail("'" + msg + "' != '" + expect + "'") self.commonChecks(nopen, pd, "{CYCLE}: cat ; cat ; cat /dev/stdin <([Pipe])")
def testTrivial(self): nopen = self.numOpenFiles() pd = ProcDag() pd.create(("true", )) pd.wait() self.commonChecks(nopen, pd, "true")
def testTrivial(self): nopen = self.numOpenFiles() pd = ProcDag() pd.create(("true",)) pd.wait() self.commonChecks(nopen, pd, "true")