def testSortPipe(self):
"pipeline command to sort a file"
class Sort(CmdRule):
def __init__(self, ifp, ofp):
CmdRule.__init__(self, requires=ifp, produces=ofp)
self.ifp = ifp
self.ofp = ofp
def run(self):
self.call(
Cmd((("sort", "-n", self.ifp), ("sort", "-nr")),
stdout=self.ofp))
er = ExRun(verbFlags=verbFlags)
ifp = er.getFile(self.getInputFile("numbers.txt"))
ofp = er.getFile(self.getOutputFile(".txt"))
er.addRule(Sort(ifp, ofp))
try:
er.run()
except Exception as ex:
prExceptions(er, ex)
raise
self.diffExpected(".txt")
self.checkGraphStates(er)
def testCmdErr(self):
"handling of pipes when process has error"
er = ExRun(verbFlags=set())
ifp = er.getFile(self.getInputFile("numbers.txt"))
ofp1 = er.getFile(self.getOutputFile(".txt.gz"))
ofp2 = er.getFile(self.getOutputFile(".txt"))
r1 = er.addCmd((["sort", "-r", FileIn(ifp)], ["tee", FileOut(ofp1)]), stdout="/dev/null")
r2 = er.addCmd((["zcat", FileIn(ofp1, autoDecompress=False)], ["false"]), stdout=FileOut(ofp2))
ex = None
try:
er.run()
except ExRunException as ex:
self.failUnlessEqual("Experiment failed: 1 error(s) encountered", str(ex))
ex1 = er.errors[0]
self.failUnless(isinstance(ex1, ExRunException))
ex2 = ex1.cause
self.failUnless(isinstance(ex2, ExRunException))
ex3 = ex2.cause
self.failUnless(isinstance(ex3, ProcException))
exre = "process exited 1: false"
self.failUnless(str(ex3),exre)
if ex == None:
self.fail("expected ProcException")
self.checkGraphStates(er,
((ofp1, ProdState.current),
(ofp2, ProdState.failed),
(r1, RuleState.ok),
(r2, RuleState.failed)))
def testSort2Rules(self):
"two commands in separate rules"
class Sort(CmdRule):
def __init__(self, ifp, ofp):
CmdRule.__init__(self, requires=ifp, produces=ofp)
self.ifp = ifp
self.ofp = ofp
def run(self):
self.call(Cmd((("sort", "-n", self.ifp), ("sort", "-nr")),
stdout=self.ofp))
class Count(CmdRule):
def __init__(self, ifp, ofp):
CmdRule.__init__(self, requires=ifp, produces=ofp)
self.ifp = ifp
self.ofp = ofp
def run(self):
self.call(Cmd((("wc", "-l"), ("sed", "-e", "s/ //g")),
stdin=self.ifp, stdout=self.ofp))
er = ExRun(verbFlags=verbFlags)
ifp = er.getFile(self.getInputFile("numbers.txt"))
ofp1 = er.getFile(self.getOutputFile(".txt"))
ofp2 = er.getFile(self.getOutputFile(".linecnt"))
er.addRule(Count(ofp1, ofp2))
er.addRule(Sort(ifp, ofp1))
try:
er.run()
except Exception as ex:
prExceptions(er, ex)
raise
self.diffExpected(".txt")
self.diffExpected(".linecnt")
self.checkGraphStates(er)
def testCmdErr(self):
"handling of pipes when process has error"
er = ExRun(verbFlags=set())
ifp = er.getFile(self.getInputFile("numbers.txt"))
ofp1 = er.getFile(self.getOutputFile(".txt.gz"))
ofp2 = er.getFile(self.getOutputFile(".txt"))
r1 = er.addCmd(
(["sort", "-r", FileIn(ifp)], ["tee", FileOut(ofp1)]),
stdout="/dev/null")
r2 = er.addCmd(
(["zcat", FileIn(ofp1, autoDecompress=False)], ["false"]),
stdout=FileOut(ofp2))
ex = None
try:
er.run()
except ExRunException as ex:
self.failUnlessEqual("Experiment failed: 1 error(s) encountered",
str(ex))
ex1 = er.errors[0]
self.failUnless(isinstance(ex1, ExRunException))
ex2 = ex1.cause
self.failUnless(isinstance(ex2, ExRunException))
ex3 = ex2.cause
self.failUnless(isinstance(ex3, ProcException))
exre = "process exited 1: false"
self.failUnless(str(ex3), exre)
if ex == None:
self.fail("expected ProcException")
self.checkGraphStates(er, ((ofp1, ProdState.current),
(ofp2, ProdState.failed),
(r1, RuleState.ok), (r2, RuleState.failed)))
def testSort2(self):
"two commands"
class Sort(CmdRule):
def __init__(self, ifp, ofp1, ofp2):
CmdRule.__init__(self, requires=ifp, produces=(ofp1, ofp2))
self.ifp = ifp
self.ofp1 = ofp1
self.ofp2 = ofp2
def run(self):
self.call(
Cmd((("sort", "-r", self.ifp), ("sort", "-nr")),
stdout=self.ofp1))
self.call(
Cmd((("wc", "-l"), ("sed", "-e", "s/ //g")),
stdin=self.ofp1,
stdout=self.ofp2))
er = ExRun(verbFlags=verbFlags)
ifp = er.getFile(self.getInputFile("numbers.txt"))
ofp1 = er.getFile(self.getOutputFile(".txt"))
ofp2 = er.getFile(self.getOutputFile(".linecnt"))
er.addRule(Sort(ifp, ofp1, ofp2))
try:
er.run()
except Exception as ex:
prExceptions(er, ex)
raise
self.diffExpected(".txt")
self.diffExpected(".linecnt")
self.checkGraphStates(er)
def __mkDependOnNoDepend(self, priFile, secFile, secContents):
ex = ExRun(verbFlags=verbFlags)
priFp = ex.getFile(priFile)
secFp = ex.getFile(secFile)
ex.addCmd(["touch", FileOut(priFp)])
ex.addCmd(["echo", secContents], requires=priFp, stdout=secFp)
ex.run()
return ex
def __mkDependOnNoDepend(self, priFile, secFile, secContents):
ex = ExRun(verbFlags=verbFlags)
priFp = ex.getFile(priFile)
secFp = ex.getFile(secFile)
ex.addCmd(["touch", FileOut(priFp)])
ex.addCmd(["echo", secContents], requires=priFp, stdout=secFp)
ex.run()
return ex
def testUptodateProd(self):
"check that a rule is not run for a current production"
er = ExRun(verbFlags=verbFlags)
prod = CurrentProd("neverRunProd")
er.addProd(prod)
rule = NeverRunRule("neverRunRule", produces=prod)
er.obtainDefaultTarget(prod)
er.addRule(rule)
er.run()
self.checkGraphStates(er)
def testUptodateProd(self):
"check that a rule is not run for a current production"
er = ExRun(verbFlags=verbFlags)
prod = CurrentProd("neverRunProd")
er.addProd(prod)
rule = NeverRunRule("neverRunRule", produces=prod)
er.obtainDefaultTarget(prod)
er.addRule(rule)
er.run()
self.checkGraphStates(er)
def testTargetLowLevel12(self):
"target runs two low level rules"
er = ExRun(verbFlags=verbFlags)
tl = self.twoLevelSetup(er, makeTargets=True)
er.run(targets=[tl.low1Tar, tl.low2Tar.name])
self.failUnlessEqual(tl.low1Rule.touchCnt, 3)
tl.low1Pset.check()
self.failUnlessEqual(tl.low2Rule.touchCnt, 3)
tl.low2Pset.check()
self.failUnlessEqual(tl.topRule.touchCnt, 0)
self.checkGraphStates(er, ((tl.topPset.prods, ProdState.outdated),
(tl.topRule, RuleState.outdated)))
def testCmdSigPipe(self):
"test command recieving SIGPIPE with no error"
er = ExRun(verbFlags=verbFlags)
ofp = er.getFile(self.getOutputFile(".txt"))
er.addCmd((["yes"], ["true"]), stdout=FileOut(ofp))
ex = None
try:
er.run()
except Exception as ex:
prExceptions(er, ex)
raise
self.checkGraphStates(er)
def testTargetTop(self):
"target runs two levels"
er = ExRun(verbFlags=verbFlags)
tl = self.twoLevelSetup(er, makeTargets=True)
er.run(targets=tl.topTar.name)
self.failUnlessEqual(tl.low1Rule.touchCnt, 3)
tl.low1Pset.check()
self.failUnlessEqual(tl.low2Rule.touchCnt, 3)
tl.low2Pset.check()
self.failUnlessEqual(tl.topRule.touchCnt, 3)
tl.topPset.check()
self.checkGraphStates(er)
def testTargetTop(self):
"target runs two levels"
er = ExRun(verbFlags=verbFlags)
tl = self.twoLevelSetup(er, makeTargets=True)
er.run(targets=tl.topTar.name)
self.failUnlessEqual(tl.low1Rule.touchCnt, 3)
tl.low1Pset.check()
self.failUnlessEqual(tl.low2Rule.touchCnt, 3)
tl.low2Pset.check()
self.failUnlessEqual(tl.topRule.touchCnt, 3)
tl.topPset.check()
self.checkGraphStates(er)
def testCmdSigPipe(self):
"test command recieving SIGPIPE with no error"
er = ExRun(verbFlags=verbFlags)
ofp = er.getFile(self.getOutputFile(".txt"))
er.addCmd((["yes"], ["true"]), stdout=FileOut(ofp))
ex = None
try:
er.run()
except Exception as ex:
prExceptions(er, ex)
raise
self.checkGraphStates(er)
def testTwoLevel(self):
"two levels of requirements"
er = ExRun(verbFlags=verbFlags)
tl = self.twoLevelSetup(er)
er.obtainDefaultTarget(tl.topPset.prods)
er.run()
self.failUnlessEqual(tl.low1Rule.touchCnt, 3)
tl.low1Pset.check()
self.failUnlessEqual(tl.low2Rule.touchCnt, 3)
tl.low2Pset.check()
self.failUnlessEqual(tl.topRule.touchCnt, 3)
tl.topPset.check()
self.checkGraphStates(er)
def testTwoLevel(self):
"two levels of requirements"
er = ExRun(verbFlags=verbFlags)
tl = self.twoLevelSetup(er)
er.obtainDefaultTarget(tl.topPset.prods)
er.run()
self.failUnlessEqual(tl.low1Rule.touchCnt, 3)
tl.low1Pset.check()
self.failUnlessEqual(tl.low2Rule.touchCnt, 3)
tl.low2Pset.check()
self.failUnlessEqual(tl.topRule.touchCnt, 3)
tl.topPset.check()
self.checkGraphStates(er)
def testTargetLowLevel12(self):
"target runs two low level rules"
er = ExRun(verbFlags=verbFlags)
tl = self.twoLevelSetup(er, makeTargets=True)
er.run(targets=[tl.low1Tar, tl.low2Tar.name])
self.failUnlessEqual(tl.low1Rule.touchCnt, 3)
tl.low1Pset.check()
self.failUnlessEqual(tl.low2Rule.touchCnt, 3)
tl.low2Pset.check()
self.failUnlessEqual(tl.topRule.touchCnt, 0)
self.checkGraphStates(er,
((tl.topPset.prods, ProdState.outdated),
(tl.topRule, RuleState.outdated)))
def testFilePrefix(self):
"test prefixes to FileIn/FileOut"
er = ExRun(verbFlags=verbFlags)
ifp = er.getFile(self.getInputFile("numbers.txt"))
ofp = er.getFile(self.getOutputFile(".txt"))
c = Cmd(("dd", "if=" + FileIn(ifp), "of=" + FileOut(ofp)))
er.addRule(CmdRule(c))
try:
er.run()
except Exception as ex:
prExceptions(er, ex)
raise
self.diffExpected(".txt")
self.checkGraphStates(er)
def testFilePrefix(self):
"test prefixes to FileIn/FileOut"
er = ExRun(verbFlags=verbFlags)
ifp = er.getFile(self.getInputFile("numbers.txt"))
ofp = er.getFile(self.getOutputFile(".txt"))
c = Cmd(("dd", "if="+FileIn(ifp), "of="+FileOut(ofp)))
er.addRule(CmdRule(c))
try:
er.run()
except Exception as ex:
prExceptions(er, ex)
raise
self.diffExpected(".txt")
self.checkGraphStates(er)
def testSortPipe(self):
"pipeline command to sort a file"
er = ExRun(verbFlags=verbFlags)
ifp = er.getFile(self.getInputFile("numbers.txt"))
ofp = er.getFile(self.getOutputFile(".txt"))
c = Cmd((("sort", "-r", FileIn(ifp)), ("sort", "-nr")), stdout=ofp)
er.addRule(CmdRule(c, requires=ifp, produces=ofp))
try:
er.run()
except Exception as ex:
prExceptions(er, ex)
raise
self.diffExpected(".txt")
self.checkGraphStates(er)
def testSortPipe(self):
"pipeline command to sort a file"
er = ExRun(verbFlags=verbFlags)
ifp = er.getFile(self.getInputFile("numbers.txt"))
ofp = er.getFile(self.getOutputFile(".txt"))
c = Cmd((("sort", "-r", FileIn(ifp)), ("sort", "-nr")), stdout=ofp)
er.addRule(CmdRule(c, requires=ifp, produces=ofp))
try:
er.run()
except Exception as ex:
prExceptions(er, ex)
raise
self.diffExpected(".txt")
self.checkGraphStates(er)
def testStdio(self):
er = ExRun(verbFlags=verbFlags)
ifp = er.getFile(self.getInputFile("numbers.txt"))
ofp1 = er.getFile(self.getOutputFile(".txt.gz"))
ofp2 = er.getFile(self.getOutputFile(".txt"))
er.addCmd(["sort", "-r"], stdin=ifp, stdout=ofp1)
er.addCmd((["zcat", FileIn(ofp1, autoDecompress=False)], ["sed", "-e", "s/^/= /"]), stdout=ofp2)
try:
er.run()
except Exception as ex:
prExceptions(er, ex)
raise
self.diffExpected(".txt")
self.checkGraphStates(er)
def testArgs(self):
er = ExRun(verbFlags=verbFlags)
ifp = er.getFile(self.getInputFile("numbers.txt"))
ofp1 = er.getFile(self.getOutputFile(".txt.gz"))
ofp2 = er.getFile(self.getOutputFile(".txt"))
er.addCmd((["sort", "-r", FileIn(ifp)], ["tee", FileOut(ofp1)]), stdout="/dev/null")
er.addCmd(["sed", "-e", "s/^/= /", FileIn(ofp1)], stdout=FileOut(ofp2))
try:
er.run()
except Exception as ex:
prExceptions(er, ex)
raise
self.diffExpected(".txt")
self.checkGraphStates(er)
def testArgs(self):
er = ExRun(verbFlags=verbFlags)
ifp = er.getFile(self.getInputFile("numbers.txt"))
ofp1 = er.getFile(self.getOutputFile(".txt.gz"))
ofp2 = er.getFile(self.getOutputFile(".txt"))
er.addCmd((["sort", "-r", FileIn(ifp)], ["tee", FileOut(ofp1)]),
stdout="/dev/null")
er.addCmd(["sed", "-e", "s/^/= /", FileIn(ofp1)], stdout=FileOut(ofp2))
try:
er.run()
except Exception as ex:
prExceptions(er, ex)
raise
self.diffExpected(".txt")
self.checkGraphStates(er)
def testSort1(self):
"single command to sort a file"
er = ExRun(verbFlags=verbFlags)
ifp = er.getFile(self.getInputFile("numbers.txt"))
ofp = er.getFile(self.getOutputFile(".txt"))
# auto add requires and produces
c = Cmd(("sort", "-n"), stdin=ifp, stdout=ofp)
er.addRule(CmdRule(c))
try:
er.run()
except Exception as ex:
prExceptions(er, ex)
raise
self.diffExpected(".txt")
self.checkGraphStates(er)
def testSort1(self):
"single command to sort a file"
er = ExRun(verbFlags=verbFlags)
ifp = er.getFile(self.getInputFile("numbers.txt"))
ofp = er.getFile(self.getOutputFile(".txt"))
# auto add requires and produces
c = Cmd(("sort", "-n"), stdin=ifp, stdout=ofp)
er.addRule(CmdRule(c))
try:
er.run()
except Exception as ex:
prExceptions(er, ex)
raise
self.diffExpected(".txt")
self.checkGraphStates(er)
def testStdio(self):
er = ExRun(verbFlags=verbFlags)
ifp = er.getFile(self.getInputFile("numbers.txt"))
ofp1 = er.getFile(self.getOutputFile(".txt.gz"))
ofp2 = er.getFile(self.getOutputFile(".txt"))
er.addCmd(["sort", "-r"], stdin=ifp, stdout=ofp1)
er.addCmd((["zcat", FileIn(ofp1, autoDecompress=False)
], ["sed", "-e", "s/^/= /"]),
stdout=ofp2)
try:
er.run()
except Exception as ex:
prExceptions(er, ex)
raise
self.diffExpected(".txt")
self.checkGraphStates(er)
def testSort2(self):
"two commands"
er = ExRun(verbFlags=verbFlags)
ifp = er.getFile(self.getInputFile("numbers.txt"))
ofp1 = er.getFile(self.getOutputFile(".txt"))
ofp2 = er.getFile(self.getOutputFile(".linecnt"))
c1 = Cmd((("sort", "-r", FileIn(ifp)), ("sort", "-nr")), stdout=ofp1)
c2 = Cmd((("wc", "-l"), ("sed", "-e", "s/ //g")), stdin=ofp1, stdout=ofp2)
er.addRule(CmdRule((c1, c2), requires=ifp))
try:
er.run()
except Exception as ex:
prExceptions(er, ex)
raise
self.diffExpected(".txt")
self.diffExpected(".linecnt")
self.checkGraphStates(er)
def testSort2RulesSub(self):
"two commands in separate rules, with file ref subtitution"
er = ExRun(verbFlags=verbFlags)
ifp = er.getFile(self.getInputFile("numbers.txt"))
ofp1 = er.getFile(self.getOutputFile(".txt"))
ofp2 = er.getFile(self.getOutputFile(".linecnt"))
c1 = Cmd((("sort", "-r", FileIn(ifp)), ("sort", "-nr"), ("tee", FileOut(ofp1))), stdout="/dev/null")
c2 = Cmd((("cat", FileIn(ofp1)), ("wc", "-l"), ("sed", "-e", "s/ //g"), ("tee", FileOut(ofp2))), stdout="/dev/null")
er.addRule(CmdRule(c2))
er.addRule(CmdRule(c1))
try:
er.run()
except Exception as ex:
prExceptions(er, ex)
raise
self.diffExpected(".txt")
self.diffExpected(".linecnt")
self.checkGraphStates(er)
def testCycle(self):
"entry node and cycle"
# can't build this graph due to linking sanity checks
id = self.getId()
ex = None
try:
er = ExRun(verbFlags=verbFlags)
# use id so file path doesn't vary with run directory
f1 = er.getFile(id + ".file1")
f2 = er.getFile(id + ".file2")
f3 = er.getFile(id + ".file3")
er.addRule(ErrorRule("cycle1", f1, f2))
er.addRule(ErrorRule("cycle2", f2, f3))
er.addRule(ErrorRule("cycle3", f3, f2))
er.run()
except ExRunException as ex:
self.failUnlessEqual(str(ex), "Production ErrorTests.ErrorTests.testCycle.file2 producedBy link already set")
if ex == None:
self.fail("expected ExRunException")
def testSort2(self):
"two commands"
er = ExRun(verbFlags=verbFlags)
ifp = er.getFile(self.getInputFile("numbers.txt"))
ofp1 = er.getFile(self.getOutputFile(".txt"))
ofp2 = er.getFile(self.getOutputFile(".linecnt"))
c1 = Cmd((("sort", "-r", FileIn(ifp)), ("sort", "-nr")), stdout=ofp1)
c2 = Cmd((("wc", "-l"), ("sed", "-e", "s/ //g")),
stdin=ofp1,
stdout=ofp2)
er.addRule(CmdRule((c1, c2), requires=ifp))
try:
er.run()
except Exception as ex:
prExceptions(er, ex)
raise
self.diffExpected(".txt")
self.diffExpected(".linecnt")
self.checkGraphStates(er)
def testCycleAll(self):
"all nodes in a cycle (no entry)"
id = self.getId()
ex = None
try:
er = ExRun(verbFlags=verbFlags)
# use id so file path doesn't vary with run directory
f1 = er.getFile(id + ".file1")
f2 = er.getFile(id + ".file2")
f3 = er.getFile(id + ".file3")
er.addRule(ErrorRule("cycleAll1", f1, f2))
er.addRule(ErrorRule("cycleAll2", f2, f3))
er.addRule(ErrorRule("cycleAll3", f3, f1))
er.run()
except CycleException as ex:
# order is not predictable
expect = "cycle detected:\n cycleAll3 ->\n ErrorTests.ErrorTests.testCycleAll.file3 ->\n cycleAll2 ->\n ErrorTests.ErrorTests.testCycleAll.file2 ->\n cycleAll1 ->\n ErrorTests.ErrorTests.testCycleAll.file1 ->"
self.failUnlessEqual(_sortMsg(str(ex)), _sortMsg(expect))
if ex == None:
self.fail("expected CycleException")
def testSimple(self):
"rule with no requirements "
# rule creates three files with known content
er = ExRun(verbFlags=verbFlags)
pset = ProdSet(er, self, (".out1", ".out2", ".out3"))
rule = TouchRule("simple", self, pset)
er.addRule(rule)
# use default target
er.run()
self.failUnlessEqual(rule.touchCnt, 3)
pset.check()
self.checkGraphStates(er)
# try again, nothing should be made this time
rule.touchCnt = 0
er.run()
self.failUnlessEqual(rule.touchCnt, 0)
pset.check()
self.checkGraphStates(er)
def testSimple(self):
"rule with no requirements "
# rule creates three files with known content
er = ExRun(verbFlags=verbFlags)
pset = ProdSet(er, self, (".out1", ".out2", ".out3"))
rule = TouchRule("simple", self, pset)
er.addRule(rule)
# use default target
er.run()
self.failUnlessEqual(rule.touchCnt, 3)
pset.check()
self.checkGraphStates(er)
# try again, nothing should be made this time
rule.touchCnt = 0
er.run()
self.failUnlessEqual(rule.touchCnt, 0)
pset.check()
self.checkGraphStates(er)
def testNoRule(self):
"no rule to make a production"
ex = None
try:
er = ExRun(verbFlags=verbFlags)
f1 = er.getFile(self.getOutputFile(".file1"))
f2 = er.getFile(self.getOutputFile(".file2"))
f3 = er.getFile(self.getId() + ".file3")
r1 = er.addRule(TouchRule("noRule1", self, f3, f2))
r2 = er.addRule(TouchRule("noRule2", self, f2, f1))
er.run()
except ExRunException as ex:
self.failUnlessEqual(str(ex), "No rule to build production(s): ErrorTests.ErrorTests.testNoRule.file3")
if ex == None:
self.fail("expected ExRunException")
self.checkGraphStates(er,
((f1, ProdState.blocked),
(f2, ProdState.blocked),
(f3, ProdState.bad),
(r1, RuleState.blocked),
(r2, RuleState.blocked)))
def testSort1(self):
"single command to sort a file"
class Sort(CmdRule):
def __init__(self, ifp, ofp):
CmdRule.__init__(self, requires=ifp, produces=ofp)
self.ifp = ifp
self.ofp = ofp
def run(self):
self.call(Cmd(("sort", "-n", self.ifp.getInPath()), stdout=self.ofp))
er = ExRun(verbFlags=verbFlags)
ifp = er.getFile(self.getInputFile("numbers.txt"))
ofp = er.getFile(self.getOutputFile(".txt"))
er.addRule(Sort(ifp, ofp))
try:
er.run()
except Exception as ex:
prExceptions(er, ex)
raise
self.diffExpected(".txt")
self.checkGraphStates(er)
def testSort2RulesSub(self):
"two commands in separate rules, with file ref subtitution"
er = ExRun(verbFlags=verbFlags)
ifp = er.getFile(self.getInputFile("numbers.txt"))
ofp1 = er.getFile(self.getOutputFile(".txt"))
ofp2 = er.getFile(self.getOutputFile(".linecnt"))
c1 = Cmd((("sort", "-r", FileIn(ifp)), ("sort", "-nr"),
("tee", FileOut(ofp1))),
stdout="/dev/null")
c2 = Cmd((("cat", FileIn(ofp1)), ("wc", "-l"), ("sed", "-e", "s/ //g"),
("tee", FileOut(ofp2))),
stdout="/dev/null")
er.addRule(CmdRule(c2))
er.addRule(CmdRule(c1))
try:
er.run()
except Exception as ex:
prExceptions(er, ex)
raise
self.diffExpected(".txt")
self.diffExpected(".linecnt")
self.checkGraphStates(er)