def test_shouldFinishLoggingExecutionStatisticsEvenIfSignaled(fixture): flagFile = fixture.tmpdir / "flag" syncer = fixture.conf.aSyncer("blocking").withContent( r"""#!/usr/bin/env bash if [ "$1" = sync ]; then trap 'sleep 0.1; echo trapped; exit 1' INT echo foo sleep 3 else exit 200 fi """).write() logFile = fixture.tmpdir / "logfile" rule = fixture.conf.ruleWithSched().withSynchronizer(syncer).withSchedOpts( ExecOnFailure="touch {0}".format(flagFile), LogFile=str(logFile)).write() fixture.afterSeconds(0.3, fixture.sigIntToProcessGroup) fixture.runSibtAsAProcess("execute-rule", rule.name) execution = None for i in range(100): time.sleep(0.05) execution = fixture.getSingleExecution(fixture.paths, None, rule.name) if execution.finished: break assert execution.finished assert not execution.succeeded strToTest(execution.output).shouldInclude("foo\ntrapped\n", "SIGINT") assert os.path.isfile(str(flagFile))
def test_shouldTreatLocsCorrespondingToPortsAsMinimumOptions(fixture): def locOptInfo(number): return optInfo("Loc" + str(number), types.Location) sched = fakeConfigurable("scheduler") syncer = fakeConfigurable("synchronizer", ports=[port(), port(), port(), port()], availableOptions=[ locOptInfo(1), locOptInfo(2), locOptInfo(3), locOptInfo(4) ]) loc1Through3 = { "Loc1": loc("/some-place"), "Loc2": loc("/place"), "Loc3": loc("/bar") } with pytest.raises(ConfigConsistencyException) as ex: fixture.factory.build("rulename", sched, syncer, {}, {}, mkSyncerOpts(**loc1Through3), True) strToTest(str(ex.value)).shouldIncludeInOrder("not", "minimum opt").andAlso.\ shouldInclude("Loc4") with noException(): fixture.factory.build( "rule", sched, syncer, {}, {}, mkSyncerOpts(Loc4=loc("/fourth"), **loc1Through3), True)
def shouldNotParse(optInfoType, string, *expectedPhrases, typeDesc=None): with pytest.raises(OptionParseException) as ex: parse(optInfoType, string, optName="Option") if typeDesc is not None: assert ex.value.errors[0].expectedType == typeDesc assert ex.value.errors[0].optionName == "Option" assert ex.value.errors[0].stringToParse == string strToTest(ex.value.errors[0].message).shouldInclude(*expectedPhrases)
def shouldOutputInCorrectOrder(runFunc, exitStatus): path = fixture.writeBashExecutable(r"echo foo >&2; exit " + str(exitStatus)) if raiseException: with pytest.raises(FakeException): runFunc(path) else: runFunc(path) _, stderr = capfd.readouterr() strToTest(stderr).shouldContainLinePatternsInOrder("before", "foo", str(exitStatus))
def test_shouldHaveAnOptionThatCausesItToWriteTheTabWithoutExecuting( self, fixture): assert "OutputTabFile" in fixture.optionNames fixture.init() tabFile = fixture.tmpdir / "tab" fixture.schedule([ buildScheduling("foo", OutputTabFile=localLocation(tabFile)), buildScheduling("bar", OutputTabFile=localLocation(tabFile)) ]) strToTest(tabFile.read()).shouldIncludeLinePatterns("*foo*", "*bar*")
def test_shouldFindAnErrorInAWriteLocThatContainsANonWriteLoc(self, fix): validator = self.construct() strToTest( validator.validate([ fix.validRule(), fix.mockRule("/mnt/data/foo", "/mnt/data") ])[0]).shouldIncludeInOrder("foo", "within", "/mnt") assert len(validator.validate([fix.mockRule("/src", "/src")])) > 0 assert len(validator.validate([fix.mockRule("/", "/mnt/backup")])) == 0
def test_shouldLogAnErrorIfAnExecutionFails(fixture): def subExecute(*_): return False scheduling = fixture.sysloggedScheduling(LogFile=localLocation( str(fixture.logFile)), Stderr=True) fixture.executeWithSyslogServer(subExecute, scheduling) assert b"failed" in fixture.singleSyslogPacket.message assert fixture.singleSyslogPacket.severity == "err" strToTest(fixture.logFile.read()).shouldInclude(scheduling.ruleName, "failed")
def test_shouldLogAnErrorIfAnInternalExceptionOccurs(fixture): class TestException(Exception): def __str__(self): return "abyss" def subExecute(*_): raise TestException() with pytest.raises(TestException): fixture.executeWithSyslogServer(subExecute, fixture.sysloggedScheduling()) strToTest(fixture.firstSyslogPacket.message).shouldInclude( b"internal", b"exception") assert b"abyss" in fixture.allSyslogMessages assert fixture.firstSyslogPacket.severity == "err"
def test_shouldFinishUnfinishedLoggedExecutionsAfterTheyHadToBeAbandoned( fixture): syncer = fixture.conf.aSyncer().withBashCode(r""" if [ "$1" = sync ]; then sleep 5 else exit 200; fi""").write() rule = fixture.conf.ruleWithSched("foo").withSynchronizer(syncer).write() fixture.afterSeconds(0.3, fixture.sigKill) fixture.runSibtAsAProcess("execute-rule", rule.name) execution = fixture.getSingleExecution(fixture.paths, None, rule.name) assert execution.finished assert not execution.succeeded strToTest(execution.output).shouldIncludeInOrder("error", "not", "finished")
def assertIsWrongSyntax(setting): ruleName = "bad-conf" iterToTest( fixture.check([ buildScheduling(ruleName, AllowedHours=setting) ])).shouldIncludeMatching( lambda error: strToTest(error).shouldInclude( ruleName, "syntax", "AllowedHours", setting))
def test(checkRestoredFile, _): existingFile = fixture.tmpdir / "existing-file" existingFile.write("") folder = fixture.tmpdir.mkdir("folder") makeNonEmptyDir(folder, testFileName) (fixture.tmpdir / testFileName).write("bar") checkRestoredFile(fixture.tmpdir, testFileName) sedReactiveFileName = r"new-\2&\Lfile" checkRestoredFile(fixture.tmpdir / sedReactiveFileName, "") checkRestoredFile(existingFile, "") with pytest.raises(ExternalFailureException): checkRestoredFile(folder, "") _, stderr = capfd.readouterr() strToTest(stderr).shouldInclude(testFileName) assert "could not make way" in stderr or "exists" in stderr
def test_shouldRaiseExceptionIfHostOrPathIsMissing(self): with pytest.raises(LocationInvalidException) as ex: RemoteLocation("http", "", "", "", "/bar") assert "host" in ex.value.problem assert strToTest(ex.value.stringRepresentation).shouldInclude( "http", "bar") with pytest.raises(LocationInvalidException) as ex: RemoteLocation("http", "", "host", "", "") assert "path" in ex.value.problem
def test(checkRestoredFolder, _): existingFile = fixture.tmpdir / "file" existingFile.write("") makeNonEmptyDir(fixture.tmpdir, testFolderName, innerFileName="inner") checkRestoredFolder(fixture.tmpdir, testFolderName, ignoreAdditionalFiles=True) if self.supportsRecursiveCopying: assert "inner" in os.listdir( str(fixture.tmpdir / testFolderName)) with pytest.raises(ExternalFailureException): checkRestoredFolder(existingFile, "") _, stderr = capfd.readouterr() strToTest(stderr).shouldIncludeAtLeastOneOf( "not a directory", "destination must be a directory") folderContainingFile = makeNonEmptyDir( fixture.tmpdir, "folder", innerFileName=testFolderName) linkToFolderContainingFile = fixture.tmpdir / "link1" linkToFolderContainingFile.mksymlinkto(folderContainingFile) with pytest.raises(ExternalFailureException): checkRestoredFolder(linkToFolderContainingFile, testFolderName) _, stderr = capfd.readouterr() strToTest(stderr).shouldIncludeAtLeastOneOf( "contains non-directory", "not a directory") checkRestoredFolder(fixture.tmpdir / "new", "") newFolder = fixture.tmpdir / "copy-of-loc1" checkRestoredFolder(newFolder, testFolderName, relativePathOfFileToRestore=".") linkedFolder = fixture.tmpdir.mkdir("linked-folder") link = fixture.tmpdir / "link2" link.mksymlinkto(linkedFolder) checkRestoredFolder(link, testFolderName) assert link.islink()
def test_shouldAllowSchedulersToControlRuleExecutions(fixture): def execute(execEnv, scheduling): execEnv.logger.log("something smart about {0}", scheduling.ruleName) succeeded = execEnv.runSynchronizer() return not succeeded fixture.replaceSibtSyncCallsWith( fixture.writeBashScript(r""" echo another thing exit 2""")) _, sched = fixture.conf.aSched().withExecuteFunc(execute).mock() rule = fixture.conf.ruleWithSyncer("a-rule").withScheduler(sched).write() fixture.runSibt("execute-rule", rule.name) execution = fixture.getSingleExecution(fixture.paths, None, rule.name) assert execution.succeeded is True strToTest(execution.output).shouldIncludeInOrder( "something smart about a-rule", "another thing")
def test_shouldLogSystemExceptionsInLessDetail(self, fixture): class SysException(BaseException): pass def raiseException(logger): raise SysException() with pytest.raises(SysException): fixture.log.logExecution("foo", constantTimeClock(), raiseException) iterToTest(fixture.executionsOf("foo")).shouldContainMatching( lambda execution: strToTest(execution.output).\ shouldInclude("exception").but.shouldNotInclude("traceback"))
def test_shouldWrapAllParseExceptionsItGetsInAConsistencyEx(fixture): def fail(_, opts): if "A" not in opts or opts["A"] == "3": raise OptionParseException( ["rule-foo" if "A" not in opts else "sched-bar"]) return True sched, syncer = fakeConfigurable("sched"), fakeConfigurable("syncer") fixture.parser.expectCalls( mock.callMatching("parseOptions", fail, anyNumber=True, ret={})) with pytest.raises(ConfigConsistencyException) as ex: fixture.makeReader([sched], [syncer]).readRule("rule", {}, { "Name": "sched", "A": "3" }, { "Name": "syncer", "A": "0" }, False) strToTest(ex.value.message).shouldIncludeInOrder("[Scheduler]", "sched-bar").\ andAlso.shouldIncludeInOrder("[Rule]", "rule-foo").andAlso.\ shouldNotInclude("[Synchronizer]")
def test_shouldCheckSyntaxOfScriptsWithoutExecutingThem(fixture, tmpdir): flagFile = str(tmpdir / "flag") erroneousCode = "touch {0}\n(echo foo".format(flagFile) schedulings = [ buildScheduling("touching", ExecOnFailure=erroneousCode), buildScheduling(ExecBefore="{") ] iterToTest(fixture.check(schedulings)).shouldContainMatching( lambda error: strToTest(error).shouldInclude( "unexpected", "ExecOnFailure", "syntax", "touching"), lambda error: "ExecBefore" in error) assert not os.path.isfile(flagFile)
def runSibtAsAProcess(self, *args, blockingStdout=False, stdoutReadFdClosed=False): self.processStartedSignal.clear() startTime = time.perf_counter() stdoutBuffer = io.StringIO() stderrBuffer = io.StringIO() readStdout = not blockingStdout and not stdoutReadFdClosed with ForkExecSubProcess(self._buildSibtCall() + list(args), blockingStdout, stdoutReadFdClosed) as process: self.processGroupId = process.pid self.processStartedSignal.set() stderrReader = threading.Thread(target=readStandardStream, args=(process.stderr, stderrBuffer)) stderrReader.start() if readStdout: stdoutReader = threading.Thread(target=readStandardStream, args=(process.stdout, stdoutBuffer)) stdoutReader.start() process.wait() if readStdout: stdoutReader.join() stderrReader.join() self.result = RunResult(strToTest(stdoutBuffer.getvalue()), strToTest(stderrBuffer.getvalue()), process.returncode) sys.stdout.write(stdoutBuffer.getvalue()) sys.stderr.write(stderrBuffer.getvalue()) self.finishedPid = process.pid self.secondsElapsedWhileRunning = time.perf_counter() - startTime
def test_shouldFailIfSyncersOrSchedsWithNamesOrNameOptionsThemselvesAreNotThere( fixture): scheduler = fakeConfigurable("the-sched") synchronizer = fakeConfigurable("the-syncer") reader = fixture.makeReader([scheduler], [synchronizer]) with pytest.raises(ConfigurableNotFoundException) as ex: reader.readRule("rule", {}, {}, {"Name": "synchronizer"}, True) assert strToTest(ex.value.message).shouldIncludeInOrder("option", "not") with pytest.raises(ConfigurableNotFoundException): reader.readRule("name", {}, {"Name": "no"}, {"Name": "the-syncer"}, False) with pytest.raises(ConfigurableNotFoundException) as ex: reader.readRule("name", {}, {"Name": "the-sched"}, {"Name": "no"}, False) assert ex.value.unitName == "no" assert ex.value.unitType == "synchronizer"
def checkTab(tabPath): strToTest(local(tabPath).read()).shouldIncludeLinePatterns( "3 0 no-interval*", "1 0 one-day*", "2 0 two-days*", "21 0 three-weeks*") return True
def checkTab(tabPath): strToTest(local(tabPath).read()).shouldIncludeLinePatterns( "*START_HOURS_RANGE=6-20") return True
def decoded(self): return strToTest(self.buffer.decode())
def checkTab(tab): assert len(fixture.tmpDir.listdir()) > 0 strToTest(local(tab).read()).shouldIncludeLinePatterns("*rule-id*") return True
def test_shouldPrintTheStartTimeIfTheRuleIsExecuting(self): column = LastStatus(makeFormatter()) currentExecution = execution(startTime=In1985) rule = mockRule(executing=True, currentExecution=currentExecution) strToTest(column.formatCell(rule)).shouldInclude( "since", "1985", "executing")
def shouldThrow(timestamp): fixture.functions.expectCalls( mock.callMatching("callFuzzy", lambda *_: True, ret=[timestamp])) with pytest.raises(ValueError) as ex: fixture.syncer.versionsOf({}, "foo", 1) strToTest(str(ex)).shouldInclude("timestamp", "format")
def string(self): return strToTest(self.stringBuffer)
def formattedString(self, dateTime, now=None): return strToTest(self.make(now).format(dateTime))