def test_binaryDiff(self):
with DisposableFile( "testfile1.bin" ) as f1, DisposableFile( "testfile2.bin") as f2:
with open( f1.filename(), "wb" ) as f:
f.write( bytes.fromhex( "2000 0839") )
with open( f2.filename(), "wb" ) as f:
f.write( bytes.fromhex( "2000 0838" ))
self.assertTrue(depthmaprunner.diffBinaryFiles(f1.filename(), f1.filename()))
self.assertFalse(depthmaprunner.diffBinaryFiles(f1.filename(), f2.filename()))
def test_successfulRunEmptyFile(self):
with DisposableFile("f1.csv") as f1, DisposableFile("f2.csv") as f2:
with open(f1.filename(), "w") as f:
f.write("action,average\n")
with open(f2.filename(), "w") as f:
f.write("action,average\n")
message = performancerunner.checkPerformance(
"f1.csv", "f2.csv", 1, 2)
self.assertEqual(message, "")
def test_performanceRegression(self):
with DisposableFile("f1.csv") as f1, DisposableFile("f2.csv") as f2:
with open(f1.filename(), "w") as f:
f.write("action,average\nfoo,10\n")
with open(f2.filename(), "w") as f:
f.write("action,average\nfoo,18\n")
message = performancerunner.checkPerformance(
"f1.csv", "f2.csv", 1, 2)
self.assertEqual(
message, "Performance regression: foo took 18s instead of 10s")
def test_fileLabelMismatch(self):
with DisposableFile("f1.csv") as f1, DisposableFile("f2.csv") as f2:
with open(f1.filename(), "w") as f:
f.write("action,average\nfoo,10\n")
with open(f2.filename(), "w") as f:
f.write("action,average\nbar,10\n")
message = performancerunner.checkPerformance(
"f1.csv", "f2.csv", 1, 2)
self.assertEqual(
message, "performance line mismatch: base 'foo', test 'bar'")
def test_configClass(self):
with DisposableFile("test.config") as testfile:
writeConfig(testfile.filename(), "../foo/bar")
conf = config.RegressionConfig(testfile.filename())
self.assertEqual(len(conf.testcases), 1)
self.assertEqual(conf.rundir, "../foo/bar")
self.assertTrue("test1" in conf.testcases)
def test_RegressionTestRunnerOneRunFails(self):
with DisposableFile("testconfig.json") as f, DisposableDirectory(
"testrundir") as d:
writeConfig(f.filename(), d.name())
runner = RegressionTestRunner(f.filename(),
lambda w, a: self.runfunc(w, a))
self.counter = 2
self.assertFalse(runner.run())
def test_RegressionTestRunnerAllGoesWell(self):
with DisposableFile("testconfig.json") as f, DisposableDirectory(
"testrundir") as d:
writeConfig(f.filename(), d.name())
runner = RegressionTestRunner(f.filename(),
lambda w, a: self.runfunc(w, a))
self.counter = -1
self.assertTrue(runner.run())
def test_filesMissing(self):
message = performancerunner.checkPerformance("f1.csv", "f2.csv", 1, 2)
self.assertEqual(message,
"Base performance timing file f1.csv is missing")
with DisposableFile("f1.csv") as f1:
with open(f1.filename(), "w") as f:
f.write("action,average\n")
message = performancerunner.checkPerformance(
"f1.csv", "f2.csv", 1, 2)
self.assertEqual(message,
"Test performance timing file f2.csv is missing")
with DisposableFile("f2.csv") as f2:
with open(f2.filename(), "w") as f:
f.write("action,average\n")
message = performancerunner.checkPerformance(
"f1.csv", "f2.csv", 1, 2)
self.assertEqual(message,
"Base performance timing file f1.csv is missing")
def test_fileLineNumberMismatch(self):
with DisposableFile("f1.csv") as f1, DisposableFile("f2.csv") as f2:
with open(f1.filename(), "w") as f:
f.write("action,average\nfoo,10\n")
with open(f2.filename(), "w") as f:
f.write("action,average\n")
message = performancerunner.checkPerformance(
"f1.csv", "f2.csv", 1, 2)
self.assertEqual(
message,
"baseline performance file f1.csv has more lines than the test one f2.csv"
)
with DisposableFile("f1.csv") as f1, DisposableFile("f2.csv") as f2:
with open(f1.filename(), "w") as f:
f.write("action,average\nfoo,10\n")
with open(f2.filename(), "w") as f:
f.write("action,average\nfoo,10\nbar,20\n")
message = performancerunner.checkPerformance(
"f1.csv", "f2.csv", 1, 2)
self.assertEqual(
message,
"baseline performance file f1.csv has fewer lines than the test one f2.csv"
)
