def test_shouldFinishLoggingExecutionStatisticsEvenIfSignaled(fixture):
flagFile = fixture.tmpdir / "flag"
syncer = fixture.conf.aSyncer("blocking").withContent(
r"""#!/usr/bin/env bash
if [ "$1" = sync ]; then
trap 'sleep 0.1; echo trapped; exit 1' INT
echo foo
sleep 3
else
exit 200
fi
""").write()
logFile = fixture.tmpdir / "logfile"
rule = fixture.conf.ruleWithSched().withSynchronizer(syncer).withSchedOpts(
ExecOnFailure="touch {0}".format(flagFile),
LogFile=str(logFile)).write()
fixture.afterSeconds(0.3, fixture.sigIntToProcessGroup)
fixture.runSibtAsAProcess("execute-rule", rule.name)
execution = None
for i in range(100):
time.sleep(0.05)
execution = fixture.getSingleExecution(fixture.paths, None, rule.name)
if execution.finished:
break
assert execution.finished
assert not execution.succeeded
strToTest(execution.output).shouldInclude("foo\ntrapped\n", "SIGINT")
assert os.path.isfile(str(flagFile))
def test_shouldTreatLocsCorrespondingToPortsAsMinimumOptions(fixture):
def locOptInfo(number):
return optInfo("Loc" + str(number), types.Location)
sched = fakeConfigurable("scheduler")
syncer = fakeConfigurable("synchronizer",
ports=[port(), port(),
port(), port()],
availableOptions=[
locOptInfo(1),
locOptInfo(2),
locOptInfo(3),
locOptInfo(4)
])
loc1Through3 = {
"Loc1": loc("/some-place"),
"Loc2": loc("/place"),
"Loc3": loc("/bar")
}
with pytest.raises(ConfigConsistencyException) as ex:
fixture.factory.build("rulename", sched, syncer, {}, {},
mkSyncerOpts(**loc1Through3), True)
strToTest(str(ex.value)).shouldIncludeInOrder("not", "minimum opt").andAlso.\
shouldInclude("Loc4")
with noException():
fixture.factory.build(
"rule", sched, syncer, {}, {},
mkSyncerOpts(Loc4=loc("/fourth"), **loc1Through3), True)
def shouldNotParse(optInfoType, string, *expectedPhrases, typeDesc=None):
with pytest.raises(OptionParseException) as ex:
parse(optInfoType, string, optName="Option")
if typeDesc is not None:
assert ex.value.errors[0].expectedType == typeDesc
assert ex.value.errors[0].optionName == "Option"
assert ex.value.errors[0].stringToParse == string
strToTest(ex.value.errors[0].message).shouldInclude(*expectedPhrases)
def shouldOutputInCorrectOrder(runFunc, exitStatus):
path = fixture.writeBashExecutable(r"echo foo >&2; exit " + str(exitStatus))
if raiseException:
with pytest.raises(FakeException):
runFunc(path)
else:
runFunc(path)
_, stderr = capfd.readouterr()
strToTest(stderr).shouldContainLinePatternsInOrder("before", "foo",
str(exitStatus))
def test_shouldHaveAnOptionThatCausesItToWriteTheTabWithoutExecuting(
self, fixture):
assert "OutputTabFile" in fixture.optionNames
fixture.init()
tabFile = fixture.tmpdir / "tab"
fixture.schedule([
buildScheduling("foo", OutputTabFile=localLocation(tabFile)),
buildScheduling("bar", OutputTabFile=localLocation(tabFile))
])
strToTest(tabFile.read()).shouldIncludeLinePatterns("*foo*", "*bar*")
def test_shouldFindAnErrorInAWriteLocThatContainsANonWriteLoc(self, fix):
validator = self.construct()
strToTest(
validator.validate([
fix.validRule(),
fix.mockRule("/mnt/data/foo", "/mnt/data")
])[0]).shouldIncludeInOrder("foo", "within", "/mnt")
assert len(validator.validate([fix.mockRule("/src", "/src")])) > 0
assert len(validator.validate([fix.mockRule("/", "/mnt/backup")])) == 0
def test_shouldLogAnErrorIfAnExecutionFails(fixture):
def subExecute(*_):
return False
scheduling = fixture.sysloggedScheduling(LogFile=localLocation(
str(fixture.logFile)),
Stderr=True)
fixture.executeWithSyslogServer(subExecute, scheduling)
assert b"failed" in fixture.singleSyslogPacket.message
assert fixture.singleSyslogPacket.severity == "err"
strToTest(fixture.logFile.read()).shouldInclude(scheduling.ruleName,
"failed")
def test_shouldLogAnErrorIfAnInternalExceptionOccurs(fixture):
class TestException(Exception):
def __str__(self):
return "abyss"
def subExecute(*_):
raise TestException()
with pytest.raises(TestException):
fixture.executeWithSyslogServer(subExecute,
fixture.sysloggedScheduling())
strToTest(fixture.firstSyslogPacket.message).shouldInclude(
b"internal", b"exception")
assert b"abyss" in fixture.allSyslogMessages
assert fixture.firstSyslogPacket.severity == "err"
def test_shouldFinishUnfinishedLoggedExecutionsAfterTheyHadToBeAbandoned(
fixture):
syncer = fixture.conf.aSyncer().withBashCode(r"""
if [ "$1" = sync ]; then
sleep 5
else exit 200; fi""").write()
rule = fixture.conf.ruleWithSched("foo").withSynchronizer(syncer).write()
fixture.afterSeconds(0.3, fixture.sigKill)
fixture.runSibtAsAProcess("execute-rule", rule.name)
execution = fixture.getSingleExecution(fixture.paths, None, rule.name)
assert execution.finished
assert not execution.succeeded
strToTest(execution.output).shouldIncludeInOrder("error", "not",
"finished")
def assertIsWrongSyntax(setting):
ruleName = "bad-conf"
iterToTest(
fixture.check([
buildScheduling(ruleName, AllowedHours=setting)
])).shouldIncludeMatching(
lambda error: strToTest(error).shouldInclude(
ruleName, "syntax", "AllowedHours", setting))
def test(checkRestoredFile, _):
existingFile = fixture.tmpdir / "existing-file"
existingFile.write("")
folder = fixture.tmpdir.mkdir("folder")
makeNonEmptyDir(folder, testFileName)
(fixture.tmpdir / testFileName).write("bar")
checkRestoredFile(fixture.tmpdir, testFileName)
sedReactiveFileName = r"new-\2&\Lfile"
checkRestoredFile(fixture.tmpdir / sedReactiveFileName, "")
checkRestoredFile(existingFile, "")
with pytest.raises(ExternalFailureException):
checkRestoredFile(folder, "")
_, stderr = capfd.readouterr()
strToTest(stderr).shouldInclude(testFileName)
assert "could not make way" in stderr or "exists" in stderr
def test_shouldRaiseExceptionIfHostOrPathIsMissing(self):
with pytest.raises(LocationInvalidException) as ex:
RemoteLocation("http", "", "", "", "/bar")
assert "host" in ex.value.problem
assert strToTest(ex.value.stringRepresentation).shouldInclude(
"http", "bar")
with pytest.raises(LocationInvalidException) as ex:
RemoteLocation("http", "", "host", "", "")
assert "path" in ex.value.problem
def test(checkRestoredFolder, _):
existingFile = fixture.tmpdir / "file"
existingFile.write("")
makeNonEmptyDir(fixture.tmpdir,
testFolderName,
innerFileName="inner")
checkRestoredFolder(fixture.tmpdir,
testFolderName,
ignoreAdditionalFiles=True)
if self.supportsRecursiveCopying:
assert "inner" in os.listdir(
str(fixture.tmpdir / testFolderName))
with pytest.raises(ExternalFailureException):
checkRestoredFolder(existingFile, "")
_, stderr = capfd.readouterr()
strToTest(stderr).shouldIncludeAtLeastOneOf(
"not a directory", "destination must be a directory")
folderContainingFile = makeNonEmptyDir(
fixture.tmpdir, "folder", innerFileName=testFolderName)
linkToFolderContainingFile = fixture.tmpdir / "link1"
linkToFolderContainingFile.mksymlinkto(folderContainingFile)
with pytest.raises(ExternalFailureException):
checkRestoredFolder(linkToFolderContainingFile, testFolderName)
_, stderr = capfd.readouterr()
strToTest(stderr).shouldIncludeAtLeastOneOf(
"contains non-directory", "not a directory")
checkRestoredFolder(fixture.tmpdir / "new", "")
newFolder = fixture.tmpdir / "copy-of-loc1"
checkRestoredFolder(newFolder,
testFolderName,
relativePathOfFileToRestore=".")
linkedFolder = fixture.tmpdir.mkdir("linked-folder")
link = fixture.tmpdir / "link2"
link.mksymlinkto(linkedFolder)
checkRestoredFolder(link, testFolderName)
assert link.islink()
def test_shouldAllowSchedulersToControlRuleExecutions(fixture):
def execute(execEnv, scheduling):
execEnv.logger.log("something smart about {0}", scheduling.ruleName)
succeeded = execEnv.runSynchronizer()
return not succeeded
fixture.replaceSibtSyncCallsWith(
fixture.writeBashScript(r"""
echo another thing
exit 2"""))
_, sched = fixture.conf.aSched().withExecuteFunc(execute).mock()
rule = fixture.conf.ruleWithSyncer("a-rule").withScheduler(sched).write()
fixture.runSibt("execute-rule", rule.name)
execution = fixture.getSingleExecution(fixture.paths, None, rule.name)
assert execution.succeeded is True
strToTest(execution.output).shouldIncludeInOrder(
"something smart about a-rule", "another thing")
def test_shouldLogSystemExceptionsInLessDetail(self, fixture):
class SysException(BaseException):
pass
def raiseException(logger):
raise SysException()
with pytest.raises(SysException):
fixture.log.logExecution("foo", constantTimeClock(), raiseException)
iterToTest(fixture.executionsOf("foo")).shouldContainMatching(
lambda execution: strToTest(execution.output).\
shouldInclude("exception").but.shouldNotInclude("traceback"))
def test_shouldWrapAllParseExceptionsItGetsInAConsistencyEx(fixture):
def fail(_, opts):
if "A" not in opts or opts["A"] == "3":
raise OptionParseException(
["rule-foo" if "A" not in opts else "sched-bar"])
return True
sched, syncer = fakeConfigurable("sched"), fakeConfigurable("syncer")
fixture.parser.expectCalls(
mock.callMatching("parseOptions", fail, anyNumber=True, ret={}))
with pytest.raises(ConfigConsistencyException) as ex:
fixture.makeReader([sched], [syncer]).readRule("rule", {}, {
"Name": "sched",
"A": "3"
}, {
"Name": "syncer",
"A": "0"
}, False)
strToTest(ex.value.message).shouldIncludeInOrder("[Scheduler]", "sched-bar").\
andAlso.shouldIncludeInOrder("[Rule]", "rule-foo").andAlso.\
shouldNotInclude("[Synchronizer]")
def test_shouldCheckSyntaxOfScriptsWithoutExecutingThem(fixture, tmpdir):
flagFile = str(tmpdir / "flag")
erroneousCode = "touch {0}\n(echo foo".format(flagFile)
schedulings = [
buildScheduling("touching", ExecOnFailure=erroneousCode),
buildScheduling(ExecBefore="{")
]
iterToTest(fixture.check(schedulings)).shouldContainMatching(
lambda error: strToTest(error).shouldInclude(
"unexpected", "ExecOnFailure", "syntax", "touching"),
lambda error: "ExecBefore" in error)
assert not os.path.isfile(flagFile)
def runSibtAsAProcess(self,
*args,
blockingStdout=False,
stdoutReadFdClosed=False):
self.processStartedSignal.clear()
startTime = time.perf_counter()
stdoutBuffer = io.StringIO()
stderrBuffer = io.StringIO()
readStdout = not blockingStdout and not stdoutReadFdClosed
with ForkExecSubProcess(self._buildSibtCall() + list(args),
blockingStdout, stdoutReadFdClosed) as process:
self.processGroupId = process.pid
self.processStartedSignal.set()
stderrReader = threading.Thread(target=readStandardStream,
args=(process.stderr,
stderrBuffer))
stderrReader.start()
if readStdout:
stdoutReader = threading.Thread(target=readStandardStream,
args=(process.stdout,
stdoutBuffer))
stdoutReader.start()
process.wait()
if readStdout:
stdoutReader.join()
stderrReader.join()
self.result = RunResult(strToTest(stdoutBuffer.getvalue()),
strToTest(stderrBuffer.getvalue()),
process.returncode)
sys.stdout.write(stdoutBuffer.getvalue())
sys.stderr.write(stderrBuffer.getvalue())
self.finishedPid = process.pid
self.secondsElapsedWhileRunning = time.perf_counter() - startTime
def test_shouldFailIfSyncersOrSchedsWithNamesOrNameOptionsThemselvesAreNotThere(
fixture):
scheduler = fakeConfigurable("the-sched")
synchronizer = fakeConfigurable("the-syncer")
reader = fixture.makeReader([scheduler], [synchronizer])
with pytest.raises(ConfigurableNotFoundException) as ex:
reader.readRule("rule", {}, {}, {"Name": "synchronizer"}, True)
assert strToTest(ex.value.message).shouldIncludeInOrder("option", "not")
with pytest.raises(ConfigurableNotFoundException):
reader.readRule("name", {}, {"Name": "no"}, {"Name": "the-syncer"},
False)
with pytest.raises(ConfigurableNotFoundException) as ex:
reader.readRule("name", {}, {"Name": "the-sched"}, {"Name": "no"},
False)
assert ex.value.unitName == "no"
assert ex.value.unitType == "synchronizer"
def checkTab(tabPath):
strToTest(local(tabPath).read()).shouldIncludeLinePatterns(
"3 0 no-interval*", "1 0 one-day*", "2 0 two-days*",
"21 0 three-weeks*")
return True
def checkTab(tabPath):
strToTest(local(tabPath).read()).shouldIncludeLinePatterns(
"*START_HOURS_RANGE=6-20")
return True
def decoded(self):
return strToTest(self.buffer.decode())
def checkTab(tab):
assert len(fixture.tmpDir.listdir()) > 0
strToTest(local(tab).read()).shouldIncludeLinePatterns("*rule-id*")
return True
def test_shouldPrintTheStartTimeIfTheRuleIsExecuting(self):
column = LastStatus(makeFormatter())
currentExecution = execution(startTime=In1985)
rule = mockRule(executing=True, currentExecution=currentExecution)
strToTest(column.formatCell(rule)).shouldInclude(
"since", "1985", "executing")
def shouldThrow(timestamp):
fixture.functions.expectCalls(
mock.callMatching("callFuzzy", lambda *_: True, ret=[timestamp]))
with pytest.raises(ValueError) as ex:
fixture.syncer.versionsOf({}, "foo", 1)
strToTest(str(ex)).shouldInclude("timestamp", "format")
def string(self):
return strToTest(self.stringBuffer)
def formattedString(self, dateTime, now=None):
return strToTest(self.make(now).format(dateTime))
