def testSortPipe(self): "pipeline command to sort a file" class Sort(CmdRule): def __init__(self, ifp, ofp): CmdRule.__init__(self, requires=ifp, produces=ofp) self.ifp = ifp self.ofp = ofp def run(self): self.call( Cmd((("sort", "-n", self.ifp), ("sort", "-nr")), stdout=self.ofp)) er = ExRun(verbFlags=verbFlags) ifp = er.getFile(self.getInputFile("numbers.txt")) ofp = er.getFile(self.getOutputFile(".txt")) er.addRule(Sort(ifp, ofp)) try: er.run() except Exception as ex: prExceptions(er, ex) raise self.diffExpected(".txt") self.checkGraphStates(er)
def testSort2(self): "two commands" class Sort(CmdRule): def __init__(self, ifp, ofp1, ofp2): CmdRule.__init__(self, requires=ifp, produces=(ofp1, ofp2)) self.ifp = ifp self.ofp1 = ofp1 self.ofp2 = ofp2 def run(self): self.call( Cmd((("sort", "-r", self.ifp), ("sort", "-nr")), stdout=self.ofp1)) self.call( Cmd((("wc", "-l"), ("sed", "-e", "s/ //g")), stdin=self.ofp1, stdout=self.ofp2)) er = ExRun(verbFlags=verbFlags) ifp = er.getFile(self.getInputFile("numbers.txt")) ofp1 = er.getFile(self.getOutputFile(".txt")) ofp2 = er.getFile(self.getOutputFile(".linecnt")) er.addRule(Sort(ifp, ofp1, ofp2)) try: er.run() except Exception as ex: prExceptions(er, ex) raise self.diffExpected(".txt") self.diffExpected(".linecnt") self.checkGraphStates(er)
def testCmdErr(self): "handling of pipes when process has error" er = ExRun(verbFlags=set()) ifp = er.getFile(self.getInputFile("numbers.txt")) ofp1 = er.getFile(self.getOutputFile(".txt.gz")) ofp2 = er.getFile(self.getOutputFile(".txt")) r1 = er.addCmd( (["sort", "-r", FileIn(ifp)], ["tee", FileOut(ofp1)]), stdout="/dev/null") r2 = er.addCmd( (["zcat", FileIn(ofp1, autoDecompress=False)], ["false"]), stdout=FileOut(ofp2)) ex = None try: er.run() except ExRunException as ex: self.failUnlessEqual("Experiment failed: 1 error(s) encountered", str(ex)) ex1 = er.errors[0] self.failUnless(isinstance(ex1, ExRunException)) ex2 = ex1.cause self.failUnless(isinstance(ex2, ExRunException)) ex3 = ex2.cause self.failUnless(isinstance(ex3, ProcException)) exre = "process exited 1: false" self.failUnless(str(ex3), exre) if ex == None: self.fail("expected ProcException") self.checkGraphStates(er, ((ofp1, ProdState.current), (ofp2, ProdState.failed), (r1, RuleState.ok), (r2, RuleState.failed)))
def __mkDependOnNoDepend(self, priFile, secFile, secContents): ex = ExRun(verbFlags=verbFlags) priFp = ex.getFile(priFile) secFp = ex.getFile(secFile) ex.addCmd(["touch", FileOut(priFp)]) ex.addCmd(["echo", secContents], requires=priFp, stdout=secFp) ex.run() return ex
def __init__(self, ruleDefs, verbFlags=None, keepGoing=False): self.er = ExRun(verbFlags=verbFlags, keepGoing=keepGoing) self.graph = self.er.graph for rdef in ruleDefs: self.__addRuleDef(rdef) # set times leaf productions for p in self.graph.productions: if p.producedBy == None: p.time = time.time()
def testUptodateProd(self): "check that a rule is not run for a current production" er = ExRun(verbFlags=verbFlags) prod = CurrentProd("neverRunProd") er.addProd(prod) rule = NeverRunRule("neverRunRule", produces=prod) er.obtainDefaultTarget(prod) er.addRule(rule) er.run() self.checkGraphStates(er)
def testTargetLowLevel12(self): "target runs two low level rules" er = ExRun(verbFlags=verbFlags) tl = self.twoLevelSetup(er, makeTargets=True) er.run(targets=[tl.low1Tar, tl.low2Tar.name]) self.failUnlessEqual(tl.low1Rule.touchCnt, 3) tl.low1Pset.check() self.failUnlessEqual(tl.low2Rule.touchCnt, 3) tl.low2Pset.check() self.failUnlessEqual(tl.topRule.touchCnt, 0) self.checkGraphStates(er, ((tl.topPset.prods, ProdState.outdated), (tl.topRule, RuleState.outdated)))
def testTargetTop(self): "target runs two levels" er = ExRun(verbFlags=verbFlags) tl = self.twoLevelSetup(er, makeTargets=True) er.run(targets=tl.topTar.name) self.failUnlessEqual(tl.low1Rule.touchCnt, 3) tl.low1Pset.check() self.failUnlessEqual(tl.low2Rule.touchCnt, 3) tl.low2Pset.check() self.failUnlessEqual(tl.topRule.touchCnt, 3) tl.topPset.check() self.checkGraphStates(er)
def testCmdSigPipe(self): "test command recieving SIGPIPE with no error" er = ExRun(verbFlags=verbFlags) ofp = er.getFile(self.getOutputFile(".txt")) er.addCmd((["yes"], ["true"]), stdout=FileOut(ofp)) ex = None try: er.run() except Exception as ex: prExceptions(er, ex) raise self.checkGraphStates(er)
def testTwoLevel(self): "two levels of requirements" er = ExRun(verbFlags=verbFlags) tl = self.twoLevelSetup(er) er.obtainDefaultTarget(tl.topPset.prods) er.run() self.failUnlessEqual(tl.low1Rule.touchCnt, 3) tl.low1Pset.check() self.failUnlessEqual(tl.low2Rule.touchCnt, 3) tl.low2Pset.check() self.failUnlessEqual(tl.topRule.touchCnt, 3) tl.topPset.check() self.checkGraphStates(er)
def testFilePrefix(self): "test prefixes to FileIn/FileOut" er = ExRun(verbFlags=verbFlags) ifp = er.getFile(self.getInputFile("numbers.txt")) ofp = er.getFile(self.getOutputFile(".txt")) c = Cmd(("dd", "if=" + FileIn(ifp), "of=" + FileOut(ofp))) er.addRule(CmdRule(c)) try: er.run() except Exception as ex: prExceptions(er, ex) raise self.diffExpected(".txt") self.checkGraphStates(er)
def testSortPipe(self): "pipeline command to sort a file" er = ExRun(verbFlags=verbFlags) ifp = er.getFile(self.getInputFile("numbers.txt")) ofp = er.getFile(self.getOutputFile(".txt")) c = Cmd((("sort", "-r", FileIn(ifp)), ("sort", "-nr")), stdout=ofp) er.addRule(CmdRule(c, requires=ifp, produces=ofp)) try: er.run() except Exception as ex: prExceptions(er, ex) raise self.diffExpected(".txt") self.checkGraphStates(er)
def testArgs(self): er = ExRun(verbFlags=verbFlags) ifp = er.getFile(self.getInputFile("numbers.txt")) ofp1 = er.getFile(self.getOutputFile(".txt.gz")) ofp2 = er.getFile(self.getOutputFile(".txt")) er.addCmd((["sort", "-r", FileIn(ifp)], ["tee", FileOut(ofp1)]), stdout="/dev/null") er.addCmd(["sed", "-e", "s/^/= /", FileIn(ofp1)], stdout=FileOut(ofp2)) try: er.run() except Exception as ex: prExceptions(er, ex) raise self.diffExpected(".txt") self.checkGraphStates(er)
def testSort1(self): "single command to sort a file" er = ExRun(verbFlags=verbFlags) ifp = er.getFile(self.getInputFile("numbers.txt")) ofp = er.getFile(self.getOutputFile(".txt")) # auto add requires and produces c = Cmd(("sort", "-n"), stdin=ifp, stdout=ofp) er.addRule(CmdRule(c)) try: er.run() except Exception as ex: prExceptions(er, ex) raise self.diffExpected(".txt") self.checkGraphStates(er)
def testStdio(self): er = ExRun(verbFlags=verbFlags) ifp = er.getFile(self.getInputFile("numbers.txt")) ofp1 = er.getFile(self.getOutputFile(".txt.gz")) ofp2 = er.getFile(self.getOutputFile(".txt")) er.addCmd(["sort", "-r"], stdin=ifp, stdout=ofp1) er.addCmd((["zcat", FileIn(ofp1, autoDecompress=False) ], ["sed", "-e", "s/^/= /"]), stdout=ofp2) try: er.run() except Exception as ex: prExceptions(er, ex) raise self.diffExpected(".txt") self.checkGraphStates(er)
def testSort2(self): "two commands" er = ExRun(verbFlags=verbFlags) ifp = er.getFile(self.getInputFile("numbers.txt")) ofp1 = er.getFile(self.getOutputFile(".txt")) ofp2 = er.getFile(self.getOutputFile(".linecnt")) c1 = Cmd((("sort", "-r", FileIn(ifp)), ("sort", "-nr")), stdout=ofp1) c2 = Cmd((("wc", "-l"), ("sed", "-e", "s/ //g")), stdin=ofp1, stdout=ofp2) er.addRule(CmdRule((c1, c2), requires=ifp)) try: er.run() except Exception as ex: prExceptions(er, ex) raise self.diffExpected(".txt") self.diffExpected(".linecnt") self.checkGraphStates(er)
def testCycle(self): "entry node and cycle" # can't build this graph due to linking sanity checks id = self.getId() ex = None try: er = ExRun(verbFlags=verbFlags) # use id so file path doesn't vary with run directory f1 = er.getFile(id + ".file1") f2 = er.getFile(id + ".file2") f3 = er.getFile(id + ".file3") er.addRule(ErrorRule("cycle1", f1, f2)) er.addRule(ErrorRule("cycle2", f2, f3)) er.addRule(ErrorRule("cycle3", f3, f2)) er.run() except ExRunException as ex: self.failUnlessEqual(str(ex), "Production ErrorTests.ErrorTests.testCycle.file2 producedBy link already set") if ex == None: self.fail("expected ExRunException")
def testSimple(self): "rule with no requirements " # rule creates three files with known content er = ExRun(verbFlags=verbFlags) pset = ProdSet(er, self, (".out1", ".out2", ".out3")) rule = TouchRule("simple", self, pset) er.addRule(rule) # use default target er.run() self.failUnlessEqual(rule.touchCnt, 3) pset.check() self.checkGraphStates(er) # try again, nothing should be made this time rule.touchCnt = 0 er.run() self.failUnlessEqual(rule.touchCnt, 0) pset.check() self.checkGraphStates(er)
def testCycleAll(self): "all nodes in a cycle (no entry)" id = self.getId() ex = None try: er = ExRun(verbFlags=verbFlags) # use id so file path doesn't vary with run directory f1 = er.getFile(id + ".file1") f2 = er.getFile(id + ".file2") f3 = er.getFile(id + ".file3") er.addRule(ErrorRule("cycleAll1", f1, f2)) er.addRule(ErrorRule("cycleAll2", f2, f3)) er.addRule(ErrorRule("cycleAll3", f3, f1)) er.run() except CycleException as ex: # order is not predictable expect = "cycle detected:\n cycleAll3 ->\n ErrorTests.ErrorTests.testCycleAll.file3 ->\n cycleAll2 ->\n ErrorTests.ErrorTests.testCycleAll.file2 ->\n cycleAll1 ->\n ErrorTests.ErrorTests.testCycleAll.file1 ->" self.failUnlessEqual(_sortMsg(str(ex)), _sortMsg(expect)) if ex == None: self.fail("expected CycleException")
def testNoRule(self): "no rule to make a production" ex = None try: er = ExRun(verbFlags=verbFlags) f1 = er.getFile(self.getOutputFile(".file1")) f2 = er.getFile(self.getOutputFile(".file2")) f3 = er.getFile(self.getId() + ".file3") r1 = er.addRule(TouchRule("noRule1", self, f3, f2)) r2 = er.addRule(TouchRule("noRule2", self, f2, f1)) er.run() except ExRunException as ex: self.failUnlessEqual(str(ex), "No rule to build production(s): ErrorTests.ErrorTests.testNoRule.file3") if ex == None: self.fail("expected ExRunException") self.checkGraphStates(er, ((f1, ProdState.blocked), (f2, ProdState.blocked), (f3, ProdState.bad), (r1, RuleState.blocked), (r2, RuleState.blocked)))
def testSort2RulesSub(self): "two commands in separate rules, with file ref subtitution" er = ExRun(verbFlags=verbFlags) ifp = er.getFile(self.getInputFile("numbers.txt")) ofp1 = er.getFile(self.getOutputFile(".txt")) ofp2 = er.getFile(self.getOutputFile(".linecnt")) c1 = Cmd((("sort", "-r", FileIn(ifp)), ("sort", "-nr"), ("tee", FileOut(ofp1))), stdout="/dev/null") c2 = Cmd((("cat", FileIn(ofp1)), ("wc", "-l"), ("sed", "-e", "s/ //g"), ("tee", FileOut(ofp2))), stdout="/dev/null") er.addRule(CmdRule(c2)) er.addRule(CmdRule(c1)) try: er.run() except Exception as ex: prExceptions(er, ex) raise self.diffExpected(".txt") self.diffExpected(".linecnt") self.checkGraphStates(er)