Esempio n. 1
0
 def testCmdErr(self):
     "handling of pipes when process has error"
     er = ExRun(verbFlags=set())
     ifp = er.getFile(self.getInputFile("numbers.txt"))
     ofp1 = er.getFile(self.getOutputFile(".txt.gz"))
     ofp2 = er.getFile(self.getOutputFile(".txt"))
     r1 = er.addCmd(
         (["sort", "-r", FileIn(ifp)], ["tee", FileOut(ofp1)]),
         stdout="/dev/null")
     r2 = er.addCmd(
         (["zcat", FileIn(ofp1, autoDecompress=False)], ["false"]),
         stdout=FileOut(ofp2))
     ex = None
     try:
         er.run()
     except ExRunException, ex:
         self.failUnlessEqual("Experiment failed: 1 error(s) encountered",
                              str(ex))
         ex1 = er.errors[0]
         self.failUnless(isinstance(ex1, ExRunException))
         ex2 = ex1.cause
         self.failUnless(isinstance(ex2, ExRunException))
         ex3 = ex2.cause
         self.failUnless(isinstance(ex3, ProcException))
         exre = "process exited 1: false"
         self.failUnless(str(ex3), exre)
Esempio n. 2
0
    def testSort2(self):
        "two commands"

        class Sort(CmdRule):
            def __init__(self, ifp, ofp1, ofp2):
                CmdRule.__init__(self, requires=ifp, produces=(ofp1, ofp2))
                self.ifp = ifp
                self.ofp1 = ofp1
                self.ofp2 = ofp2

            def run(self):
                self.call(
                    Cmd((("sort", "-r", self.ifp), ("sort", "-nr")),
                        stdout=self.ofp1))
                self.call(
                    Cmd((("wc", "-l"), ("sed", "-e", "s/ //g")),
                        stdin=self.ofp1,
                        stdout=self.ofp2))

        er = ExRun(verbFlags=verbFlags)
        ifp = er.getFile(self.getInputFile("numbers.txt"))
        ofp1 = er.getFile(self.getOutputFile(".txt"))
        ofp2 = er.getFile(self.getOutputFile(".linecnt"))
        er.addRule(Sort(ifp, ofp1, ofp2))
        try:
            er.run()
        except Exception, ex:
            prExceptions(er, ex)
            raise
Esempio n. 3
0
    def testSort2Rules(self):
        "two commands in separate rules"
        class Sort(CmdRule):
            def __init__(self, ifp, ofp):
                CmdRule.__init__(self, requires=ifp, produces=ofp)
                self.ifp = ifp
                self.ofp = ofp
            def run(self):
                self.call(Cmd((("sort", "-n", self.ifp), ("sort", "-nr")),
                              stdout=self.ofp))

        class Count(CmdRule):
            def __init__(self, ifp, ofp):
                CmdRule.__init__(self, requires=ifp, produces=ofp)
                self.ifp = ifp
                self.ofp = ofp
            def run(self):
                self.call(Cmd((("wc", "-l"), ("sed", "-e", "s/ //g")),
                              stdin=self.ifp, stdout=self.ofp))


        er = ExRun(verbFlags=verbFlags)
        ifp = er.getFile(self.getInputFile("numbers.txt"))
        ofp1 = er.getFile(self.getOutputFile(".txt"))
        ofp2 = er.getFile(self.getOutputFile(".linecnt"))
        er.addRule(Count(ofp1, ofp2))
        er.addRule(Sort(ifp, ofp1))
        try:
            er.run()
        except Exception, ex:
            prExceptions(er, ex)
            raise
Esempio n. 4
0
 def testFilePrefix(self):
     "test prefixes to FileIn/FileOut"
     er = ExRun(verbFlags=verbFlags)
     ifp = er.getFile(self.getInputFile("numbers.txt"))
     ofp = er.getFile(self.getOutputFile(".txt"))
     c = Cmd(("dd", "if=" + FileIn(ifp), "of=" + FileOut(ofp)))
     er.addRule(CmdRule(c))
     try:
         er.run()
     except Exception, ex:
         prExceptions(er, ex)
         raise
Esempio n. 5
0
 def testArgs(self):
     er = ExRun(verbFlags=verbFlags)
     ifp = er.getFile(self.getInputFile("numbers.txt"))
     ofp1 = er.getFile(self.getOutputFile(".txt.gz"))
     ofp2 = er.getFile(self.getOutputFile(".txt"))
     er.addCmd((["sort", "-r", FileIn(ifp)], ["tee", FileOut(ofp1)]), stdout="/dev/null")
     er.addCmd(["sed", "-e", "s/^/= /", FileIn(ofp1)], stdout=FileOut(ofp2))
     try:
         er.run()
     except Exception, ex:
         prExceptions(er, ex)
         raise
Esempio n. 6
0
 def testFilePrefix(self):
     "test prefixes to FileIn/FileOut"
     er = ExRun(verbFlags=verbFlags)
     ifp = er.getFile(self.getInputFile("numbers.txt"))
     ofp = er.getFile(self.getOutputFile(".txt"))
     c = Cmd(("dd", "if="+FileIn(ifp), "of="+FileOut(ofp)))
     er.addRule(CmdRule(c))
     try:
         er.run()
     except Exception, ex:
         prExceptions(er, ex)
         raise
Esempio n. 7
0
 def testStdio(self):
     er = ExRun(verbFlags=verbFlags)
     ifp = er.getFile(self.getInputFile("numbers.txt"))
     ofp1 = er.getFile(self.getOutputFile(".txt.gz"))
     ofp2 = er.getFile(self.getOutputFile(".txt"))
     er.addCmd(["sort", "-r"], stdin=ifp, stdout=ofp1)
     er.addCmd((["zcat", FileIn(ofp1, autoDecompress=False)], ["sed", "-e", "s/^/= /"]), stdout=ofp2)
     try:
         er.run()
     except Exception, ex:
         prExceptions(er, ex)
         raise
Esempio n. 8
0
 def testSortPipe(self):
     "pipeline command to sort a file"
     er = ExRun(verbFlags=verbFlags)
     ifp = er.getFile(self.getInputFile("numbers.txt"))
     ofp = er.getFile(self.getOutputFile(".txt"))
     c = Cmd((("sort", "-r", FileIn(ifp)), ("sort", "-nr")), stdout=ofp)
     er.addRule(CmdRule(c, requires=ifp, produces=ofp))
     try:
         er.run()
     except Exception, ex:
         prExceptions(er, ex)
         raise
Esempio n. 9
0
 def testSortPipe(self):
     "pipeline command to sort a file"
     er = ExRun(verbFlags=verbFlags)
     ifp = er.getFile(self.getInputFile("numbers.txt"))
     ofp = er.getFile(self.getOutputFile(".txt"))
     c = Cmd((("sort", "-r", FileIn(ifp)), ("sort", "-nr")), stdout=ofp)
     er.addRule(CmdRule(c, requires=ifp, produces=ofp))
     try:
         er.run()
     except Exception, ex:
         prExceptions(er, ex)
         raise
Esempio n. 10
0
 def testNoRule(self):
     "no rule to make a production"
     ex = None
     try:
         er = ExRun(verbFlags=verbFlags)
         f1 = er.getFile(self.getOutputFile(".file1"))
         f2 = er.getFile(self.getOutputFile(".file2"))
         f3 = er.getFile(self.getId() + ".file3")
         r1 = er.addRule(TouchRule("noRule1", self, f3, f2))
         r2 = er.addRule(TouchRule("noRule2", self, f2, f1))
         er.run()
     except ExRunException, ex:
         self.failUnlessEqual(str(ex), "No rule to build production(s): ErrorTests.ErrorTests.testNoRule.file3")
Esempio n. 11
0
 def testSort1(self):
     "single command to sort a file"
     er = ExRun(verbFlags=verbFlags)
     ifp = er.getFile(self.getInputFile("numbers.txt"))
     ofp = er.getFile(self.getOutputFile(".txt"))
     # auto add requires and produces
     c = Cmd(("sort", "-n"), stdin=ifp, stdout=ofp)
     er.addRule(CmdRule(c))
     try:
         er.run()
     except Exception, ex:
         prExceptions(er, ex)
         raise
Esempio n. 12
0
 def testSort1(self):
     "single command to sort a file"
     er = ExRun(verbFlags=verbFlags)
     ifp = er.getFile(self.getInputFile("numbers.txt"))
     ofp = er.getFile(self.getOutputFile(".txt"))
     # auto add requires and produces
     c = Cmd(("sort", "-n"), stdin=ifp, stdout=ofp)
     er.addRule(CmdRule(c))
     try:
         er.run()
     except Exception, ex:
         prExceptions(er, ex)
         raise
Esempio n. 13
0
 def testArgs(self):
     er = ExRun(verbFlags=verbFlags)
     ifp = er.getFile(self.getInputFile("numbers.txt"))
     ofp1 = er.getFile(self.getOutputFile(".txt.gz"))
     ofp2 = er.getFile(self.getOutputFile(".txt"))
     er.addCmd((["sort", "-r", FileIn(ifp)], ["tee", FileOut(ofp1)]),
               stdout="/dev/null")
     er.addCmd(["sed", "-e", "s/^/= /", FileIn(ofp1)], stdout=FileOut(ofp2))
     try:
         er.run()
     except Exception, ex:
         prExceptions(er, ex)
         raise
Esempio n. 14
0
 def testSort2(self):
     "two commands"
     er = ExRun(verbFlags=verbFlags)
     ifp = er.getFile(self.getInputFile("numbers.txt"))
     ofp1 = er.getFile(self.getOutputFile(".txt"))
     ofp2 = er.getFile(self.getOutputFile(".linecnt"))
     c1 = Cmd((("sort", "-r", FileIn(ifp)), ("sort", "-nr")), stdout=ofp1)
     c2 = Cmd((("wc", "-l"), ("sed", "-e", "s/ //g")), stdin=ofp1, stdout=ofp2)
     er.addRule(CmdRule((c1, c2), requires=ifp))
     try:
         er.run()
     except Exception, ex:
         prExceptions(er, ex)
         raise
Esempio n. 15
0
 def testStdio(self):
     er = ExRun(verbFlags=verbFlags)
     ifp = er.getFile(self.getInputFile("numbers.txt"))
     ofp1 = er.getFile(self.getOutputFile(".txt.gz"))
     ofp2 = er.getFile(self.getOutputFile(".txt"))
     er.addCmd(["sort", "-r"], stdin=ifp, stdout=ofp1)
     er.addCmd((["zcat", FileIn(ofp1, autoDecompress=False)
                 ], ["sed", "-e", "s/^/= /"]),
               stdout=ofp2)
     try:
         er.run()
     except Exception, ex:
         prExceptions(er, ex)
         raise
Esempio n. 16
0
 def testCycle(self):
     "entry node and cycle"
     # can't build this graph due to linking sanity checks
     id = self.getId()
     with self.assertRaises(ExRunException) as cm:
         er = ExRun(verbFlags=verbFlags)
         # use id so file path doesn't vary with run directory
         f1 = er.getFile(id + ".file1")
         f2 = er.getFile(id + ".file2")
         f3 = er.getFile(id + ".file3")
         er.addRule(ErrorRule("cycle1", f1, f2))
         er.addRule(ErrorRule("cycle2", f2, f3))
         er.addRule(ErrorRule("cycle3", f3, f2))
         er.run()
     self.assertEqual(str(cm.exception), "Production errorTests.ErrorTests.testCycle.file2 producedBy link already set")
Esempio n. 17
0
 def testSort2RulesSub(self):
     "two commands in separate rules, with file ref subtitution"
     er = ExRun(verbFlags=verbFlags)
     ifp = er.getFile(self.getInputFile("numbers.txt"))
     ofp1 = er.getFile(self.getOutputFile(".txt"))
     ofp2 = er.getFile(self.getOutputFile(".linecnt"))
     c1 = Cmd((("sort", "-r", FileIn(ifp)), ("sort", "-nr"), ("tee", FileOut(ofp1))), stdout="/dev/null")
     c2 = Cmd((("cat", FileIn(ofp1)), ("wc", "-l"), ("sed", "-e", "s/ //g"), ("tee", FileOut(ofp2))), stdout="/dev/null")
     er.addRule(CmdRule(c2))
     er.addRule(CmdRule(c1))
     try:
         er.run()
     except Exception, ex:
         prExceptions(er, ex)
         raise
Esempio n. 18
0
 def testSort2(self):
     "two commands"
     er = ExRun(verbFlags=verbFlags)
     ifp = er.getFile(self.getInputFile("numbers.txt"))
     ofp1 = er.getFile(self.getOutputFile(".txt"))
     ofp2 = er.getFile(self.getOutputFile(".linecnt"))
     c1 = Cmd((("sort", "-r", FileIn(ifp)), ("sort", "-nr")), stdout=ofp1)
     c2 = Cmd((("wc", "-l"), ("sed", "-e", "s/ //g")),
              stdin=ofp1,
              stdout=ofp2)
     er.addRule(CmdRule((c1, c2), requires=ifp))
     try:
         er.run()
     except Exception, ex:
         prExceptions(er, ex)
         raise
Esempio n. 19
0
 def testCycleAll(self):
     "all nodes in a cycle (no entry)"
     id = self.getId()
     with self.assertRaises(CycleException) as cm:
         er = ExRun(verbFlags=verbFlags)
         # use id so file path doesn't vary with run directory
         f1 = er.getFile(id + ".file1")
         f2 = er.getFile(id + ".file2")
         f3 = er.getFile(id + ".file3")
         er.addRule(ErrorRule("cycleAll1", f1, f2))
         er.addRule(ErrorRule("cycleAll2", f2, f3))
         er.addRule(ErrorRule("cycleAll3", f3, f1))
         er.run()
     # order is not predictable
     expect = "cycle detected:\n  cycleAll3 ->\n  errorTests.ErrorTests.testCycleAll.file3 ->\n  cycleAll2 ->\n  errorTests.ErrorTests.testCycleAll.file2 ->\n  cycleAll1 ->\n  errorTests.ErrorTests.testCycleAll.file1 ->"
     self.assertEqual(_sortMsg(str(cm.exception)), _sortMsg(expect))
Esempio n. 20
0
 def testNoRule(self):
     "no rule to make a production"
     ex = None
     try:
         er = ExRun(verbFlags=verbFlags)
         f1 = er.getFile(self.getOutputFile(".file1"))
         f2 = er.getFile(self.getOutputFile(".file2"))
         f3 = er.getFile(self.getId() + ".file3")
         r1 = er.addRule(TouchRule("noRule1", self, f3, f2))
         r2 = er.addRule(TouchRule("noRule2", self, f2, f1))
         er.run()
     except ExRunException, ex:
         self.failUnlessEqual(
             str(ex),
             "No rule to build production(s): ErrorTests.ErrorTests.testNoRule.file3"
         )
Esempio n. 21
0
 def __mkDependOnNoDepend(self, priFile, secFile, secContents):
     ex = ExRun(verbFlags=verbFlags)
     priFp = ex.getFile(priFile)
     secFp = ex.getFile(secFile)
     ex.addCmd(["touch", FileOut(priFp)])
     ex.addCmd(["echo", secContents], requires=priFp, stdout=secFp)
     ex.run()
     return ex
Esempio n. 22
0
 def testNoRule(self):
     "no rule to make a production"
     with self.assertRaises(ExRunException) as cm:
         er = ExRun(verbFlags=verbFlags)
         f1 = er.getFile(self.getOutputFile(".file1"))
         f2 = er.getFile(self.getOutputFile(".file2"))
         f3 = er.getFile(self.getId() + ".file3")
         r1 = er.addRule(TouchRule("noRule1", self, f3, f2))
         r2 = er.addRule(TouchRule("noRule2", self, f2, f1))
         er.run()
     self.assertEqual(str(cm.exception), "No rule to build production(s): errorTests.ErrorTests.testNoRule.file3")
     self.checkGraphStates(er,
                           ((f1, ProdState.blocked),
                            (f2, ProdState.blocked),
                            (f3, ProdState.bad),
                            (r1, RuleState.blocked),
                            (r2, RuleState.blocked)))
Esempio n. 23
0
 def __mkDependOnNoDepend(priFile, secFile, secContents):
     ex = ExRun(verbFlags=verbFlags)
     priFp = ex.getFile(priFile)
     secFp = ex.getFile(secFile)
     ex.addCmd(["touch", FileOut(priFp)])
     ex.addCmd(["echo", secContents], requires=priFp, stdout=secFp)
     ex.run()
     return ex
Esempio n. 24
0
 def testCycle(self):
     "entry node and cycle"
     # can't build this graph due to linking sanity checks            
     id = self.getId()
     ex = None
     try:
         er = ExRun(verbFlags=verbFlags)
         # use id so file path doesn't vary with run directory
         f1 = er.getFile(id + ".file1")
         f2 = er.getFile(id + ".file2")
         f3 = er.getFile(id + ".file3")
         er.addRule(ErrorRule("cycle1", f1, f2))
         er.addRule(ErrorRule("cycle2", f2, f3))
         er.addRule(ErrorRule("cycle3", f3, f2))
         er.run()
     except ExRunException, ex:
         self.failUnlessEqual(str(ex), "Production ErrorTests.ErrorTests.testCycle.file2 producedBy link already set")
Esempio n. 25
0
 def testCycleAll(self):
     "all nodes in a cycle (no entry)"
     id = self.getId()
     ex = None
     try:
         er = ExRun(verbFlags=verbFlags)
         # use id so file path doesn't vary with run directory
         f1 = er.getFile(id + ".file1")
         f2 = er.getFile(id + ".file2")
         f3 = er.getFile(id + ".file3")
         er.addRule(ErrorRule("cycleAll1", f1, f2))
         er.addRule(ErrorRule("cycleAll2", f2, f3))
         er.addRule(ErrorRule("cycleAll3", f3, f1))
         er.run()
     except CycleException, ex:
         # order is not predictable
         expect = "cycle detected:\n  cycleAll3 ->\n  ErrorTests.ErrorTests.testCycleAll.file3 ->\n  cycleAll2 ->\n  ErrorTests.ErrorTests.testCycleAll.file2 ->\n  cycleAll1 ->\n  ErrorTests.ErrorTests.testCycleAll.file1 ->"
         self.failUnlessEqual(_sortMsg(str(ex)), _sortMsg(expect))
Esempio n. 26
0
 def testSort2RulesSub(self):
     "two commands in separate rules, with file ref subtitution"
     er = ExRun(verbFlags=verbFlags)
     ifp = er.getFile(self.getInputFile("numbers.txt"))
     ofp1 = er.getFile(self.getOutputFile(".txt"))
     ofp2 = er.getFile(self.getOutputFile(".linecnt"))
     c1 = Cmd((("sort", "-r", FileIn(ifp)), ("sort", "-nr"),
               ("tee", FileOut(ofp1))),
              stdout="/dev/null")
     c2 = Cmd((("cat", FileIn(ofp1)), ("wc", "-l"), ("sed", "-e", "s/ //g"),
               ("tee", FileOut(ofp2))),
              stdout="/dev/null")
     er.addRule(CmdRule(c2))
     er.addRule(CmdRule(c1))
     try:
         er.run()
     except Exception, ex:
         prExceptions(er, ex)
         raise
Esempio n. 27
0
    def testSort1(self):
        "single command to sort a file"

        class Sort(CmdRule):
            def __init__(self, ifp, ofp):
                CmdRule.__init__(self, requires=ifp, produces=ofp)
                self.ifp = ifp
                self.ofp = ofp
            def run(self):
                self.call(Cmd(("sort", "-n", self.ifp.getInPath()), stdout=self.ofp))
        er = ExRun(verbFlags=verbFlags)
        ifp = er.getFile(self.getInputFile("numbers.txt"))
        ofp = er.getFile(self.getOutputFile(".txt"))
        er.addRule(Sort(ifp, ofp))
        try:
            er.run()
        except Exception, ex:
            prExceptions(er, ex)
            raise
Esempio n. 28
0
 def testCycle(self):
     "entry node and cycle"
     # can't build this graph due to linking sanity checks
     id = self.getId()
     ex = None
     try:
         er = ExRun(verbFlags=verbFlags)
         # use id so file path doesn't vary with run directory
         f1 = er.getFile(id + ".file1")
         f2 = er.getFile(id + ".file2")
         f3 = er.getFile(id + ".file3")
         er.addRule(ErrorRule("cycle1", f1, f2))
         er.addRule(ErrorRule("cycle2", f2, f3))
         er.addRule(ErrorRule("cycle3", f3, f2))
         er.run()
     except ExRunException, ex:
         self.failUnlessEqual(
             str(ex),
             "Production ErrorTests.ErrorTests.testCycle.file2 producedBy link already set"
         )
Esempio n. 29
0
 def testCmdSigPipe(self):
     "test command recieving SIGPIPE with no error"
     er = ExRun(verbFlags=verbFlags)
     ofp = er.getFile(self.getOutputFile(".txt"))
     er.addCmd((["yes"], ["true"]), stdout=FileOut(ofp))
     ex = None
     try:
         er.run()
     except Exception, ex:
         prExceptions(er, ex)
         raise
Esempio n. 30
0
 def testCmdSigPipe(self):
     "test command recieving SIGPIPE with no error"
     er = ExRun(verbFlags=verbFlags)
     ofp = er.getFile(self.getOutputFile(".txt"))
     er.addCmd((["yes"], ["true"]), stdout=FileOut(ofp))
     ex = None
     try:
         er.run()
     except Exception, ex:
         prExceptions(er, ex)
         raise
Esempio n. 31
0
 def testCmdErr(self):
     "handling of pipes when process has error"
     er = ExRun(verbFlags=set())
     ifp = er.getFile(self.getInputFile("numbers.txt"))
     ofp1 = er.getFile(self.getOutputFile(".txt.gz"))
     ofp2 = er.getFile(self.getOutputFile(".txt"))
     r1 = er.addCmd((["sort", "-r", FileIn(ifp)], ["tee", FileOut(ofp1)]), stdout="/dev/null")
     r2 = er.addCmd((["zcat", FileIn(ofp1, autoDecompress=False)], ["false"]), stdout=FileOut(ofp2))
     ex = None
     try:
         er.run()
     except ExRunException, ex:
         self.failUnlessEqual("Experiment failed: 1 error(s) encountered", str(ex))
         ex1 = er.errors[0]
         self.failUnless(isinstance(ex1, ExRunException))
         ex2 = ex1.cause
         self.failUnless(isinstance(ex2, ExRunException))
         ex3 = ex2.cause
         self.failUnless(isinstance(ex3, ProcException))
         exre = "process exited 1: false"
         self.failUnless(str(ex3),exre)
Esempio n. 32
0
    def testSort1(self):
        "single command to sort a file"

        class Sort(CmdRule):
            def __init__(self, ifp, ofp):
                CmdRule.__init__(self, requires=ifp, produces=ofp)
                self.ifp = ifp
                self.ofp = ofp

            def run(self):
                self.call(
                    Cmd(("sort", "-n", self.ifp.getInPath()), stdout=self.ofp))

        er = ExRun(verbFlags=verbFlags)
        ifp = er.getFile(self.getInputFile("numbers.txt"))
        ofp = er.getFile(self.getOutputFile(".txt"))
        er.addRule(Sort(ifp, ofp))
        try:
            er.run()
        except Exception, ex:
            prExceptions(er, ex)
            raise
Esempio n. 33
0
 def testCmdErr(self):
     "handling of pipes when process has error"
     er = ExRun(verbFlags=set())
     ifp = er.getFile(self.getInputFile("numbers.txt"))
     ofp1 = er.getFile(self.getOutputFile(".txt.gz"))
     ofp2 = er.getFile(self.getOutputFile(".txt"))
     r1 = er.addCmd((["sort", "-r", FileIn(ifp)], ["tee", FileOut(ofp1)]), stdout="/dev/null")
     r2 = er.addCmd((["zcat", FileIn(ofp1, autoDecompress=False)], ["false"]), stdout=FileOut(ofp2))
     with self.assertRaises(ExRunException) as cm:
         er.run()
     self.assertEqual("Experiment failed: 1 error(s) encountered", str(cm.exception))
     ex1 = er.errors[0]
     self.assertTrue(isinstance(ex1, ExRunException))
     ex2 = ex1.cause
     self.assertTrue(isinstance(ex2, ExRunException))
     ex3 = ex2.cause
     self.assertTrue(isinstance(ex3, ProcException))
     exre = "process exited 1: false"
     self.assertTrue(str(ex3),exre)
     self.checkGraphStates(er,
                           ((ofp1, ProdState.current),
                            (ofp2, ProdState.failed),
                            (r1,   RuleState.ok),
                            (r2,   RuleState.failed)))