class TestCaseTest(TestCase):
def setUp(self):
self.result = TestResult()
def testTemplateMethod(self):
test = WasRun("testMethod")
test.run(self.result)
assert("setUp testMethod tearDown " == test.log)
def testResult(self):
test = WasRun("testMethod")
test.run(self.result)
assert("1 run, 0 failed" == result.summary())
def testFailedResult(self):
test = WasRun("testBrokenMethod")
test.run(self.result)
assert("1 run, 1 failed" == result.summary())
def testFailedResultFormatting(self):
self.result.testStarted()
self.result.testFaield()
assert("1 run, 1 failed" == result.summary())
def testSuite(self):
suite = TestSuite()
suite.add(WasRun("testMethod"))
suite.add(WasRun("testBrokenMethod"))
suite.run(self.result)
assert("2 run, 1 failed" == result.summary())
class TestCaseTest(TestCase):
def setUp(self):
self.result = TestResult()
def testTemplateMethod(self):
test = WasRun("testMethod")
test.run(self.result)
assert("setUp testMethod tearDown " == test.log)
def testResult(self):
test = WasRun("testMethod")
test.run(self.result)
assert("1 run, 0 failed" == result.summary())
def testFailedResult(self):
test = WasRun("testBrokenMethod")
test.run(self.result)
assert("1 run, 1 failed", result.summary)
def testFailedResultFormatting(self):
self.result.testStarted()
self.result.testFailed()
assert("1 run, 1 failed" == result.summary())
def testSuite(self):
suite = TestSuite()
suite.add(WasRun("testMethod"))
suite.add(WasRun("testBrokenMethod"))
suite.run(self.result)
assert("2 run, 1 failed" == result.summary())
def run(self):
ExeRunnerBase.run(self)
if (not self.doDetailedChecks) or self.statSum == 0:
return
## Temporary work-around until either TP conversion added for
## Muon SDOs or acmd diff-root can be made to ignore them.
reLine = 'Found [0] different leaves'
comment = 'ORIGINAL RESULT OVERRIDDEN'
with open(self.outputFile) as ifile:
for line in ifile:
if reLine in line:
self.testResults[0] = TestResult(cmd=self.cmd,
status=0,
comment=comment)
## Klobber statSum, as we want the test to return success.
self.statSum = 0
return
self.statSum += 1
self.testResults[0] = TestResult(cmd=self.cmd,
status=1,
comment=comment)
return
def testTestSuite(self):
suite = TestSuite()
suite.add(WasRun("testMethod"))
suite.add(WasRun("testBrokenMethod"))
result = TestResult()
suite.run(result)
print "???"
assert result.summary() == "2 run, 1 failed"
def main():
suite = TestSuite()
suite.add(TestCaseTest("test_template_method"))
suite.add(TestCaseTest("test_result"))
suite.add(TestCaseTest("test_failed_result"))
suite.add(TestCaseTest("test_failed_result_formatting"))
suite.add(TestCaseTest("test_suite"))
result = TestResult()
suite.run(result)
print(result.summary())
def test_record_result():
tr1 = TestResult(False, 'failed', 'failed detail')
tr2 = TestResult(True, 'testing', 'testing detail')
tr3 = None
r = Report()
r.record_result(tr1)
r.record_result(tr2)
r.record_result(tr3)
assert len(r.results) == 2
assert r.results[0] is tr1
assert r.results[1] is tr2
r.log()
def run(self):
result = TestResult()
result.testStarted()
self.setUp()
method = getattr(self, self.name)
try:
method()
except:
result.testFailed()
self.tearDown()
return result
class TestCaseTest(TestCase):
def set_up(self):
self.result = TestResult()
def test_template_method(self):
test = WasRun("test_method")
test.run(self.result)
assert "set_up test_method tear_down " == test.log
def test_result(self):
test = WasRun("test_method")
test.run(self.result)
assert "1 run, 0 failed" == self.result.summary()
def test_failed_result(self):
test = WasRun("test_broken_method")
test.run(self.result)
assert "1 run, 1 failed" == self.result.summary()
def test_failed_result_formatting(self):
self.result.test_started()
self.result.test_failed()
assert "1 run, 1 failed" == self.result.summary()
def test_failed_setup_result(self):
test = SetupFails("test_method")
test.run(self.result)
assert "1 run, 1 failed" == self.result.summary()
def test_failed_method_runs_teardown(self):
test = TestMethodFails("test_method")
test.run(self.result)
assert test.tear_down_ran is True
class TestCaseTest(TestCase):
def setUp(self):
self.result = TestResult()
def testTemplateMethod(self):
test = WasRun('testMethod')
test.run(self.result)
assert('setUp testMethod tearDown ' == test.log)
def testResult(self):
test = WasRun('testMethod')
test.run(self.result)
assert('1 run, 0 failed' == self.result.summary())
def testFailedResult(self):
test = WasRun('testBrokenMethod')
test.run(self.result)
assert('1 run, 1 failed' == self.result.summary())
def testFailedResultFormatting(self):
self.result.testStarted()
self.result.testFailed()
assert('1 run, 1 failed' == self.result.summary())
def testSuite(sefl):
suite = TestSuite()
suite.add(WasRun('testMethod'))
suite.add(WasRun('testBrokenMethod'))
suite.run(self.result)
assert('2 run, 1 failed' == self.result.summary())
class TestCaseTest(TestCase):
def setup(self):
self.result = TestResult()
def test_template_method(self):
self.test = WasRun("test_method")
self.test.run(self.result)
assert ("setup test_method tear_down " == self.test.log)
def test_result(self):
self.test = WasRun("test_method")
self.test.run(self.result)
assert ("1 run, 0 failed" == self.result.summary())
def test_failed_result(self):
self.test = WasRun("test_broken_method")
self.test.run(self.result)
assert ("1 run, 1 failed" == self.result.summary())
def test_failed_result_formatting(self):
self.result.test_started()
self.result.test_failed()
assert ("1 run, 1 failed" == self.result.summary())
def test_if_tear_down_runs_regardless(self):
self.test = WasRun("test_broken_method")
self.test.run(self.result)
assert ("setup tear_down " == self.test.log)
def test_suite(self):
suite = TestSuite()
suite.add(WasRun("test_method"))
suite.add(WasRun("test_broken_method"))
suite.run(self.result)
assert ("2 run, 1 failed" == self.result.summary())
class BeckTestCaseTest(TestCase):
def setUp(self):
self.result = TestResult()
def testTemplateMethod(self):
test = WasRun("testMethod")
test.run(self.result)
assert "setUp testMethod tearDown " == test.log
def testResult(self):
test = WasRun("testMethod")
test.run(self.result)
assert "1 run. 0 failed" == self.result.summary()
def testFailedResult(self):
test = WasRun("wasBrokenMethod")
test.run(self.result)
assert "1 run. 1 failed" == self.result.summary()
def testFailedResultFormatted(self):
self.result.testStarted()
self.result.testFailed()
assert "1 run. 1 failed" == self.result.summary()
def testSuite(self):
suite = TestSuite()
suite.add(WasRun("testMethod"))
suite.add(WasRun("wasBrokenMethod"))
suite.run(self.result)
assert "2 run. 1 failed" == self.result.summary()
class TestCaseTest(TestCase):
def __init__(self, run_method_name):
super().__init__(run_method_name)
def set_up(self):
self.result = TestResult()
def test_suite(self):
suite = TestSuite()
suite.add(WasRun('test_method'))
suite.run(self.result)
assert ("2 run, 1 failed" == self.result.summary())
def test_template_method(self):
test = WasRun("test_method")
test.run(self.result)
assert ("SetUp test_method tear_down " == test.log)
def test_result(self):
test = WasRun("test_method")
test.run(self.result)
assert ("1 run, 0 failed" == self.result.summary())
def test_failed_result(self):
test = WasRun("")
test.run(self.result)
assert ("1 run, 1 failed" == self.result.summary())
def test_failed_result_formatting(self):
self.result.test_started()
self.result.test_failed()
assert ("1 run, 1 failed" == self.result.summary())
def run_test(test, env, idx):
print('[{}] Validating {}'.format(idx, test.command_line()))
# Setup files
result = open('.{}_result'.format(idx), 'w')
output = open('.{}_output'.format(idx), 'w')
env['VALIDATOR_COVERAGE'] = '.{}_coverage'.format(idx)
# Execute
process = subprocess.Popen([test.command] + test.arguments,
stdout=output,
stderr=result,
env=env)
retcode = process.wait()
output.close()
result.close()
if retcode == 0:
print('[{}] Validated {}'.format(idx, test.command_line()))
else:
print('[{}] Failed validation {}'.format(idx, test.command_line()))
# Parse test results and remove test files
with open('.{}_result'.format(idx), 'r') as f:
result = f.read()
f.close()
with open('.{}_coverage'.format(idx), 'r') as f:
coverage = f.read()
f.close()
test_result = TestResult(test, retcode, result, coverage)
os.remove('.{}_result'.format(idx))
os.remove('.{}_coverage'.format(idx))
os.remove('.{}_output'.format(idx))
return test_result
def __junit_test(self, test):
""" Runs a junit test .class file """
tr = TestResult(name=test["name"], test_type=test["type"])
junit_runner = "junit-platform-console-standalone-1.3.1.jar"
cmd = shlex.split(
(f"""java -jar {utils.libdir() / junit_runner}"""
f""" -cp "{self.outdir}" -c {test["classname"]} --reports-dir="""
f"""{self.outdir} --disable-ansi-colors"""),
posix=("win" not in sys.platform),
)
tr.retval, tr.stdout, tr.stderr = utils.run_command(cmd,
cwd=self.workdir)
tr.cmd = " ".join(cmd)
tr.stdout, tr.points, tr.maxpoints = junit.parse_xml(self.workdir /
self.outdir)
self.results.append(tr)
def __custom_test(self, test):
""" Runs any arbitrary command and saves output. """
tr = TestResult(name=test["name"],
test_type=test["type"],
cmd=test["command"])
cmd = shlex.split(tr.cmd)
tr.retval, tr.stdout, tr.stderr = utils.run_command(
cmd,
cwd=self.workdir,
sinput="",
timeout=test.get("timeout"),
memory_limit=self.config.get("memory_limit"),
process_limit=self.config.get("process_limit"),
)
self.results.append(tr)
def run(self):
for f in (self.testFile, self.refFile):
if not os.path.exists(f):
msg = '%s.run missing file %s: ' % (self.__class__.__name__, f)
self.testResults.append(TestResult(cmd=self.cmd, comment=msg))
return
ExeRunnerBase.run(self)
class TestCaseTest(TestCase):
def setUp(self):
self.result = TestResult()
def testTemplateMethod(self):
test = WasRun("testMethod")
test.run(self.result)
assert("setUp testMethod tearDown " == test.log)
def testResult(self):
test = WasRun("testMethod")
test.run(self.result)
assert("1 run, 0 failed" == self.result.summary())
def testFailedResult(self):
test = WasRun("testBrokenMethod")
test.run(self.result)
assert("1 run, 1 failed" == self.result.summary())
def testFailedResultFormatting(self):
self.result.testStarted()
self.result.testFailed()
assert("1 run, 1 failed" == self.result.summary())
def testExceptionInSetup(self):
test = BadSetup("testMethod")
test.run(self.result)
assert("1 run, 1 failed" == self.result.summary())
def testSuite(self):
suite = TestSuite.emptySuite()
suite.add(WasRun("testMethod"))
suite.add(WasRun("testBrokenMethod"))
suite.run(self.result)
assert("2 run, 1 failed" == self.result.summary())
def testSuiteFromCases(self):
test = WasRun("testMethod")
suite = TestSuite.fromTestCase(test)
assert(len(suite.tests) == 1)
suite.run(self.result)
assert("1 run, 0 failed" == self.result.summary())
class TestCaseTest(TestCase):
def setUp(self):
self.result = TestResult()
def testTemplateMethod(self):
test = WasRun("testMethod")
test.run(self.result)
assert "setUp testMethod tearDown" == test.log
def testResult(self):
test = WasRun("testMethod")
test.run(self.result)
assert "1 run, 0 failed" == self.result.summary()
def testFailedResultFormatting(self):
self.result.testStarted()
self.result.testFailed()
assert "1 run, 1 failed" == self.result.summary()
def testFailedResult(self):
self.test = WasRun("testBrokenMethod")
self.test.run(self.result)
assert "1 run, 1 failed" == self.result.summary()
def checkForLog(self):
self.test = WasRun("testMethod")
self.test.run(self.result)
assert path.isfile("C:/Nathans Python Folder/log.txt")
def testSetUpException(self):
self.test = TestCase("setUp")
self.test.run(self.result)
assert "1 run, 1 failed" == self.result.summary()
def testSuite(self):
suite = TestSuite()
suite.add(TestCaseTest("testFailedResultFormatting"))
suite.add(TestCaseTest("setUp"))
suite.add(TestCaseTest("testTemplateMethod"))
suite.add(WasRun("testMethod"))
suite.add(TestCaseTest("testResult"))
suite.add(WasRun("testBrokenMethod"))
suite.add(TestCaseTest("testSetUpException"))
suite.run(self.result)
assert "6 run, 2 failed" == self.result.summary()
def get_last_run(self, branch):
cursor = self.connection.cursor()
# Find run result
cursor.execute('select * from Runs where branch=? order by timestamp desc;', (branch,))
row = cursor.fetchone()
if row == None:
return None
# Rebuild RunResult object Run row
run_result = RunResult.from_dict({k: row[k] for k in row.keys()})
# Find test results
for row in cursor.execute('select * from Tests where run_id=?;', (row['run_id'],)):
# Rebuild TestResult object Test row
run_result.test_results.append(TestResult.from_dict({k: row[k] for k in row.keys()}))
return run_result
def runtest(config: Config, workdir: Path, results_dir: Path):
print(f"========== Grading {workdir.name}")
(workdir / config["output_dir"]).mkdir(exist_ok=True, parents=True)
secret_files = []
with (results_dir / f"{workdir.name}.txt").open("w", encoding="utf-8") as logfile:
if "build" in config:
logfile.write(utils.box_text("Build Step") + "\n")
# copy files from project root to build location
if "required_files" in config["build"]:
for file in config["build"]["required_files"]:
(workdir / file["dest"]).mkdir(exist_ok=True, parents=True)
try:
copy(
Path(".config") / file["file"], Path(workdir / file["dest"])
)
except FileNotFoundError as ex:
raise click.FileError(ex.filename, "are you sure it exists?")
if file.get("secret"):
secret_files.append(Path(workdir / file["dest"] / file["file"]))
if "commands" in config["build"]:
for command in config["build"]["commands"]:
br = TestResult(test_type="build", cmd=command)
command = shlex.split(command)
br.retval, br.stdout, br.stderr = utils.run_command(
command, cwd=workdir
)
logfile.write(br.log(config["output"]["build"]))
# loop through and run all tests
test_runner = TestRunner(logfile, workdir, config)
test_runner.run_all()
test_runner.log()
for file in secret_files:
file.unlink()
def run(self):
result = TestResult()
result.testStarted()
self.setUp()
try:
method = getattr(self, self.name)
method()
except:
result.testFailed()
self.tearDown()
return result
def run_test(test, env, idx):
print('[{}] Validating {}'.format(idx, test.command_line()))
# Setup files
result_file = '.{}_result'.format(idx)
output_file = '.{}_output'.format(idx)
coverage_file = '.{}_coverage'.format(idx)
error = False
with open(result_file, 'w') as result:
with open(output_file, 'w') as output:
env['VALIDATOR_COVERAGE'] = coverage_file
# Execute
try:
process = subprocess.Popen([test.command] + test.arguments,
stdout=output,
stderr=result,
env=env)
retcode = process.wait()
except OSError as e:
print('[{}] Failled to execute {} : {}'.format(
idx, test.command_line(), e))
retcode = 255
error = True
# Parse test results and remove test files
with open(result_file, 'r') as f:
result = f.read()
if os.path.isfile(coverage_file):
with open(coverage_file, 'r') as f:
coverage = f.read()
os.remove(coverage_file)
else:
coverage = ""
test_result = TestResult(test, retcode, result, coverage, error)
if test_result.retcode == 0:
print('[{}] Validated {}'.format(idx, test.command_line()))
else:
print('[{}] Failed validation {}'.format(idx, test.command_line()))
os.remove(result_file)
os.remove(output_file)
return test_result
def run_test(test, env, idx):
print('[{}] Validating {}'.format(idx, test.command_line()))
# Setup files
coverage_file = '.{}_coverage'.format(idx)
error = False
env['VALIDATOR_COVERAGE'] = coverage_file
# Execute
try:
process = subprocess.run([test.command] + test.arguments,
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
env=env)
retcode = process.returncode
result = process.stderr.decode('utf8')
except OSError as e:
print('[{}] Failled to execute {} : {}'.format(idx,
test.command_line(), e))
retcode = 255
error = True
result = ""
# Parse test results and remove test files
if os.path.isfile(coverage_file):
with open(coverage_file, 'rb') as f:
coverage = f.read().decode('utf8')
os.remove(coverage_file)
else:
coverage = ""
test_result = TestResult(test, retcode, result, coverage, error)
if test_result.retcode == 0:
print('[{}] Validated {}'.format(idx, test.command_line()))
else:
print('[{}] Failed validation {}'.format(idx, test.command_line()))
return test_result
def doTheTest(data, burnInN, windows, bootIter):
""" Computes the statistics, runs the bootstrap and produces the TestResult object
:param data: numpy 2D array, the last column should contain responses.
:param burnInN: the number of the first data point to be used for bootstrap
:param windows: list or tuple of window sizes
:param bootIter:
:return: TestResult
"""
burnInData = data[:burnInN, :]
kernel, gp, eps = fitKernel(burnInData,
GP(kernel=1 * RBF() + WhiteKernel()))
windowAndDoubledSizes = sorted(
list(
set([
windowOrDoubled for w in windows
for windowOrDoubled in [w, w * 2]
])))
invKernels = getInvertedKernels(data, kernel, windowAndDoubledSizes)
stats = [getStatistics(data, invKernels, window) for window in windows]
bootStats = residualBootstrap(data, invKernels, windows, eps, gp, bootIter)
crits = multiplicityCorrectedCrits(bootStats, 0.01)
return TestResult(windows, stats, crits)
def parseHeaderField(self, headerStr, checkHeaderNames):
"""Trys to genericly seperate the given line into the header field
name and the value. If the second parameter is True the header field
name is checked if is written like in the BNF of RFC3261. In case
it is not written like in the RFC3261 a sequence of TestResults
(result CAUTION) will be returned.
"""
ret = []
try:
index = headerStr.index(':')
except:
Log.logDebug(
"SipMessage.parseHeaderField(): failed to find the name-value sperator in: "
+ str(headerStr), 2)
return ret
if checkHeaderNames:
hfn_raw = headerStr[0:index].strip()
if hfn_raw.capitalize() in Helper.rfc_headers:
if not (hfn_raw in Helper.rfc_bfn_headers):
ret.append(
TestResult(
TestResult.TR_CAUT,
"header name '" + str(hfn_raw) +
"' is not exactly written as in RFC3261's BNF"))
else:
Log.logDebug(
"SipMessage.parseHeaderFiled(): header field name '" +
str(hfn_raw) + "' is not in RFC3261", 2)
hfn = Helper.getMappedHFH(headerStr[0:index].rstrip())
if (self.headerFields.has_key(hfn)
and (hfn in Helper.multiple_headers)):
self.setHeaderValue(hfn, self.getHeaderValue(hfn).rstrip() + \
", " + headerStr[index+1:].lstrip())
else:
self.setHeaderValue(hfn, headerStr[index + 1:].lstrip())
return ret
def run(self):
bad_refFile = not os.path.exists(self.refFile)
bad_newFile = not os.path.exists(self.newFile)
msg = ''
msgTpl = '%s: file does not exist, not running'
if bad_refFile:
msg += msgTpl % self.refFile
if bad_newFile:
msg += msgTpl % self.newFile
# Should we run this test?
if msg: # This will turn up on the results web page for this job
tr = TestResult(msg)
self.testResults.append(tr)
return
# all ok - call the worker script.
# You are free to call this as many times as you like.
# For example, if your worker script was grepping an input file,
# and you had many input file matches, you could loop over each file here
# in the master script and for each file call ExeRunnerBase.run().
# The advantage of this instead of passing the list of files to your worker
# is that you would get on your job web page a test result for each input file.
ExeRunnerBase.run(self)
test.run(self.result)
assert "1 run, 1 failed" == self.result.summary()
def test_failed_result_formatting(self):
self.result.test_started()
self.result.test_failed()
assert "1 run, 1 failed" == self.result.summary()
def test_failed_setup_result(self):
test = SetupFails("test_method")
test.run(self.result)
assert "1 run, 1 failed" == self.result.summary()
def test_failed_method_runs_teardown(self):
test = TestMethodFails("test_method")
test.run(self.result)
assert test.tear_down_ran is True
suite = TestSuite()
suite.add(TestCaseTest("test_template_method"))
suite.add(TestCaseTest("test_result"))
suite.add(TestCaseTest("test_failed_result_formatting"))
suite.add(TestCaseTest("test_failed_result"))
suite.add(TestCaseTest("test_failed_setup_result"))
suite.add(TestCaseTest("test_failed_method_runs_teardown"))
suiteResult = TestResult()
suite.run(suiteResult)
print(suiteResult.summary())
from TestCaseTest import TestCaseTest
from TestResult import TestResult
from TestSuite import TestSuite
suite = TestSuite()
suite.add(TestCaseTest("testTemplateMethod"))
suite.add(TestCaseTest("testResult"))
suite.add(TestCaseTest("testFailedResultFormatting"))
suite.add(TestCaseTest("testFailedResult"))
suite.add(TestCaseTest("testSuite"))
result = TestResult()
suite.run(result)
print(result.summary())
from TestCaseTest import TestCaseTest
from TestResult import TestResult
from TestSuite import TestSuite
suite = TestSuite()
suite.add(TestCaseTest("testTemplateMethod"))
suite.add(TestCaseTest("testResult"))
suite.add(TestCaseTest("testFailedResult"))
suite.add(TestCaseTest("testFailedResultFormatting"))
suite.add(TestCaseTest("testSuite"))
result = TestResult()
suite.run(result)
print result.summary()
assert("1 run, 1 failed" == self.result.summary())
def testFailedResult(self):
test = WasRun("testBrokenMethod")
test.run(self.result)
assert("1 run, 1 failed" == self.result.summary())
def testSuite(self):
suite = TestSuite()
suite.add(WasRun("testMethod"))
suite.add(WasRun("testBrokenMethod"))
suite.run(self.result)
assert("2 run, 1 failed" == self.result.summary())
if __name__ == "__main__":
testMethods = [ "testTemplateMethod",
"testResult",
"testFailedResultFormatting",
"testFailedResult",
"testSuite"
];
suite = TestSuite()
for method in testMethods:
case = TestCaseTest(method)
suite.add(case)
result = TestResult()
suite.run(result)
print("{} : {}".format(method, result.summary()))
def setUp(self):
self.result = TestResult()
def run(self, result):
result = TestResult()
for test in self.tests:
test.run(result)
def testResult(self):
test = WasRun("testMethod")
test.run(result)
assert("1 run, 0 failed" == result.summary())
def testFailedResult(self):
test = WasRun("testBrokenMethod")
test.run(result)
assert("1 run, 1 failed" == result.summary())
def testFailedResultFormatting(self):
result.testStarted()
result.testFailed()
assert("1 run, 1 failed" == result.summary())
def testSuite(self):
suite = TestSuite()
suite.add(WasRun("testMethod"))
suite.add(WasRun("testBrokenMethod"))
suite.run(result)
assert("2 run, 1 failed" == result.summary())
suite = TestSuite()
suite.add(TestCaseTest("testTemplateMethod"))
suite.add(TestCaseTest("testResult"))
suite.add(TestCaseTest("testFailedResult"))
suite.add(TestCaseTest("testFailedResultFormatting"))
suite.add(TestCaseTest("testSuite"))
suite.run(TestResult())
def setUp(self): self.result = TestResult()
from TestCaseTest import TestCaseTest as TestCaseTest
from TestSuite import TestSuite as TestSuite
from TestResult import TestResult as TestResult
suite = TestSuite()
suite.add(TestCaseTest('testTemplateMethod'))
suite.add(TestCaseTest('testResult'))
suite.add(TestCaseTest('testFailedResultFormatting'))
suite.add(TestCaseTest('testSuite'))
result = TestResult()
suite.run(result)
print(result.summary())
class TestCaseTest(TestCase):
def setUp(self):
self.result = TestResult()
def testTemplateMethod(self):
test = WasRun("testMethod")
test.run(self.result)
assert "setUp testMethod tearDown " == test.log
def testResult(self):
test = WasRun("testMethod")
test.run(self.result)
assert "1 run, 0 failed" == self.result.summary()
def testSetUpFailedResult(self):
test = SetUpError("testMethod")
test.run(self.result)
assert "1 run, 1 failed, SetUpError.testMethod-Exception" == self.result.summary(
)
def testFailedResult(self):
test = WasRun("testBrokenMethod")
test.run(self.result)
assert "1 run, 1 failed, WasRun.testBrokenMethod-Exception" == self.result.summary(
)
def testFailedResultFormatting(self):
self.result.testStarted()
self.result.testFailed(WasRun("testMethod"), Exception())
assert "1 run, 1 failed, WasRun.testMethod-Exception" == self.result.summary(
)
def testTearDownFailedResult(self):
test = TearDownError("testMethod")
test.run(self.result)
assert "1 run, 1 failed, TearDownError.testMethod-Exception" == self.result.summary(
)
def testSuite(self):
suite = TestSuite()
suite.add(WasRun("testMethod"))
suite.add(WasRun("testBrokenMethod"))
suite.run(self.result)
assert "2 run, 1 failed, WasRun.testBrokenMethod-Exception" == self.result.summary(
)
def testSetUpErrorsReport(self):
suite = TestSuite()
suite.add(WasRun("testMethod"))
suite.add(SetUpError("testMethod", ZeroDivisionError))
suite.add(SetUpError("testBrokenMethod", ValueError))
suite.run(self.result)
assert "3 run, 2 failed, SetUpError.testMethod-ZeroDivisionError, SetUpError.testBrokenMethod-ValueError" == \
self.result.summary()
def testSuiteComposite(self):
suite = TestSuiteComposite()
suite.add(WasRun())
suite.run(self.result)
assert "2 run, 1 failed, WasRun.testBrokenMethod-Exception" == \
self.result.summary()
def __diff_test(self, test):
""" Runs a specified command and compares the result to the expected output """
tr = TestResult(
name=test["name"],
test_type=test["type"],
cmd=test["command"],
maxpoints=test.get("points"),
)
inp = test.get("stdin")
if inp is None:
with (Path(".config") / test["stdinFile"]).absolute().open() as f:
inp = f.read()
out = test.get("stdout")
if out is None:
filename = test.get("stdoutFile")
if filename:
with (Path(".config") / filename).absolute().open() as f:
out = f.read()
err = test.get("stderr")
if err is None:
filename = test.get("stderrFile")
if filename:
with (Path(".config") / filename).absolute().open() as f:
err = f.read()
cmd = shlex.split(tr.cmd)
# perform globbing of executable name
if test.get("glob_command"):
executable = list(self.workdir.glob(cmd[0]))
if len(executable) > 1:
tr.warning = f"WARNING: multiple executables matched for glob {cmd[0]}"
elif len(executable) > 0:
cmd[0] = str(executable[0].absolute())
tr.cmd = "".join(cmd)
else:
tr.warning = f"WARNING: no files matched with glob {cmd[0]}"
tr.retval, tr.stdout, tr.stderr = utils.run_command(
cmd,
cwd=self.workdir,
sinput=inp,
timeout=test.get("timeout"),
memory_limit=self.config.get("memory_limit"),
process_limit=self.config.get("process_limit"),
)
tr.diffout = ""
if out:
tr.diffout += utils.diff_output(out, tr.stdout)
if err:
tr.diffout += utils.diff_output(err, tr.stderr)
# diff is blank if matches perfectly
if len(tr.diffout) == 0:
tr.points = tr.maxpoints
tr.diffout = "Output is identical"
else:
tr.points = 0
self.results.append(tr)
def run(self):
result = TestResult()
for test in tests:
test.run(result)
def create_result(self):
return TestResult(self.success, self.display_str, self.result_detail)
def testFailedResultFormatting(self):
result = TestResult()
result.testStarted()
result.testFailed()
assert result.summary() == "1 run, 1 failed"
