def testWrite(self):
outf = self.getOutputFile(".out")
outfGz = self.getOutputFile(".out.gz")
pl = Pipeline(("gzip", "-1"), "w", otherEnd=outfGz)
self.cpFileToPl("simple1.txt", pl)
pl.wait()
procOps.runProc(("zcat", outfGz), stdout=outf)
self.diffExpected(".out")
def testRead(self):
inf = self.getInputFile("simple1.txt")
infGz = self.getOutputFile(".txt.gz")
procOps.runProc(("gzip", "-c", inf), stdout=infGz)
pl = Pipeline(("gzip", "-dc"), "r", otherEnd=infGz)
self.cpPlToFile(pl, ".out")
pl.wait()
self.diffExpected(".out")
def testWrite(self):
outf = self.getOutputFile(".out")
outfGz = self.getOutputFile(".out.gz")
pl = Pipeline(("gzip", "-1"), "w", otherEnd=outfGz)
self.cpFileToPl("simple1.txt", pl)
pl.wait()
procOps.runProc(("zcat", outfGz), stdout=outf)
self.diffExpected(".out")
def testRead(self):
inf = self.getInputFile("simple1.txt")
infGz = self.getOutputFile(".txt.gz")
procOps.runProc(("gzip", "-c", inf), stdout=infGz)
pl = Pipeline(("gzip", "-dc"), "r", otherEnd=infGz)
self.cpPlToFile(pl, ".out")
pl.wait()
self.diffExpected(".out")
def testExitCode(self):
pl = Pipeline(("false", ))
e = None
try:
pl.wait()
except ProcException as e:
pass
self.failUnless(e != None)
# FIXME: should Procline keep an ordered list?
for p in pl.procs:
self.failUnless(p.returncode == 1)
def testReadMult(self):
inf = self.getInputFile("simple1.txt")
pl = Pipeline((("gzip", "-1c"), ("gzip", "-dc"), ("wc", ),
("sed", "-e", "s/ */\t/g")),
"r",
otherEnd=inf)
self.cpPlToFile(pl, ".wc")
pl.wait()
self.diffExpected(".wc")
def testExitCode(self):
pl = Pipeline(("false",))
e = None
try:
pl.wait()
except ProcException as e:
pass
self.failUnless(e != None)
# FIXME: should Procline keep an ordered list?
for p in pl.procs:
self.failUnless(p.returncode == 1)
def callProc(cmd, keepLastNewLine=False):
"""call a process and return stdout, exception with stderr in message.
The cmd is either a list of command and arguments, or pipeline, specified by
a list of lists of commands and arguments."""
stdout = Pipeline.DataReader()
pl = Pipeline.Procline(cmd, stdin="/dev/null", stdout=stdout)
pl.wait()
out = stdout.get()
if (not keepLastNewLine) and (len(out) > 0) and (out[-1] == "\n"):
out = out[0:-1]
return out
def XXtestPassWrite(self):
"using FIFO to pass pipe to another process for writing"
# FIXME: should this be supported somehow
inf = self.getInputFile("simple1.txt")
outf = self.getOutputFile(".out")
pipePath = self.getOutputFile(".fifo")
pl = Pipeline(("sort", "-r"), "w", otherEnd=outf, pipePath=pipePath)
procOps.runProc(["cat"], stdin=inf, stdout=pl.pipePath)
pl.wait()
self.diffExpected(".out")
def testWriteMult(self):
outf = self.getOutputFile(".wc")
# grr, BSD wc adds an extract space, so just convert to tabs
pl = Pipeline((("gzip", "-1"), ("gzip", "-dc"), ("wc", ),
("sed", "-e", "s/ */\t/g")),
"w",
otherEnd=outf)
self.cpFileToPl("simple1.txt", pl)
pl.wait()
self.diffExpected(".wc")
def XXtestPassWrite(self):
"using FIFO to pass pipe to another process for writing"
# FIXME: should this be supported somehow
inf = self.getInputFile("simple1.txt")
outf = self.getOutputFile(".out")
pipePath = self.getOutputFile(".fifo")
pl = Pipeline(("sort", "-r"), "w", otherEnd=outf, pipePath=pipePath)
procOps.runProc(["cat"], stdin=inf, stdout=pl.pipePath)
pl.wait()
self.diffExpected(".out")
def testReadMult(self):
inf = self.getInputFile("simple1.txt")
pl = Pipeline((("gzip","-1c"),
("gzip", "-dc"),
("wc",),
("sed", "-e", "s/ */\t/g")),
"r", otherEnd=inf)
self.cpPlToFile(pl, ".wc")
pl.wait()
self.diffExpected(".wc")
def XXtestPassRead(self):
"using FIFO to pass pipe to another process for reading"
# FIXME: should this be supported somehow
inf = self.getInputFile("simple1.txt")
infGz = self.getOutputFile(".txt.gz")
cpOut = self.getOutputFile(".out")
procOps.runProc(("gzip", "-c", inf), stdout=infGz)
pl = Pipeline(("gzip", "-dc"), "r", otherEnd=infGz)
procOps.runProc(["cat"], stdin=pl.pipePath, stdout=cpOut)
pl.wait()
self.diffExpected(".out")
def testWriteMult(self):
outf = self.getOutputFile(".wc")
# grr, BSD wc adds an extract space, so just convert to tabs
pl = Pipeline((("gzip", "-1"),
("gzip", "-dc"),
("wc",),
("sed", "-e", "s/ */\t/g")),
"w", otherEnd=outf)
self.cpFileToPl("simple1.txt", pl)
pl.wait()
self.diffExpected(".wc")
def XXtestPassRead(self):
"using FIFO to pass pipe to another process for reading"
# FIXME: should this be supported somehow
inf = self.getInputFile("simple1.txt")
infGz = self.getOutputFile(".txt.gz")
cpOut = self.getOutputFile(".out")
procOps.runProc(("gzip", "-c", inf), stdout=infGz)
pl = Pipeline(("gzip", "-dc"), "r", otherEnd=infGz)
procOps.runProc(["cat"], stdin=pl.pipePath, stdout=cpOut)
pl.wait()
self.diffExpected(".out")
def __getOutput(self, pdag, fspec):
"""Get an output file. If fspec can be None, File or FileOut, or any
object that can be converted to a string. Return the appropriate Pout
object, None, or a string. Adds compression process if needed."""
if fspec == None:
return None
if isinstance(fspec, File):
fspec = FileOut(fspec)
if not isinstance(fspec, FileOut):
return str(fspec)
# handle File object, include arg prefix
path = fspec.file.getOutPath()
if fileOps.isCompressed(path) and fspec.autoCompress:
pdev = Pipeline.Pipe()
pdag.create((fileOps.compressCmd(path), ), stdin=pdev, stdout=path)
return Pipeline.POut(pdev, fspec.prefix)
else:
return Pipeline.POut(Pipeline.File(path), fspec.prefix)
def __createProcDag(self):
"construct ProcDag for command"
pdag = Pipeline.ProcDag()
prevStdout = self.__getInput(pdag, self.stdin)
stderr = self.__getOutput(pdag, self.stderr)
last = self[-1]
for scmd in self:
prevStdout = self.__createProc(pdag, scmd, prevStdout, stderr,
(scmd == last))
return pdag
def runProc(cmd, stdin="/dev/null", stdout=None, stderr=None, noError=False):
"""run a process, with I/O redirection to specified file paths or open
file objects. None specifies inheriting. If noError is True, then
the exit code is returned rather than generating an error"""
# FIXME: drop noError???
pl = Pipeline.Procline(cmd, stdin=stdin, stdout=stdout, stderr=stderr)
code = 0
try:
pl.wait()
except Pipeline.ProcException as ex:
code = ex.returncode
if not noError:
raise
return code
def __createProc(self, pdag, scmd, stdin, stderr, isLast):
"define one process in the command, return stdout"
if isLast:
stdout = self.__getOutput(pdag, self.stdout)
else:
stdout = Pipeline.Pipe()
# convert arguments
pcmd = []
for a in scmd:
if isinstance(a, FileIn):
pcmd.append(self.__getInput(pdag, a))
elif isinstance(a, FileOut):
pcmd.append(self.__getOutput(pdag, a))
else:
pcmd.append(str(a))
pdag.create(pcmd, stdin, stdout, stderr)
return stdout
def testSigPipe(self):
"test not reading all of pipe output"
pl = Pipeline([("yes", ), ("true", )], "r")
pl.wait()
def testSigPipe(self):
"test not reading all of pipe output"
pl = Pipeline([("yes",), ("true",)], "r")
pl.wait()