def test_run_with_memory_output(self):
port = MockPort()
test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test')
memory_results = {'Malloc': 10, 'JSHeap': 5}
self.maxDiff = None
driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test, memory_results)
output_capture = OutputCapture()
output_capture.capture_output()
try:
metrics = test._run_with_driver(driver, None)
finally:
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
self.assertEqual(actual_stdout, '')
self.assertEqual(actual_stderr, '')
self.assertEqual(actual_logs, '')
self.assertEqual(len(metrics), 3)
self.assertEqual(metrics[0].metric(), 'Time')
self.assertEqual(metrics[0].to_dict(), {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms',
'values': [float(i * 1000) for i in range(2, 21)]})
self.assertEqual(metrics[1].metric(), 'Malloc')
self.assertEqual(metrics[1].to_dict(), {'max': 10, 'avg': 10.0, 'median': 10, 'min': 10, 'stdev': 0.0, 'unit': 'bytes',
'values': [float(10)] * 19})
self.assertEqual(metrics[2].metric(), 'JSHeap')
self.assertEqual(metrics[2].to_dict(), {'max': 5, 'avg': 5.0, 'median': 5, 'min': 5, 'stdev': 0.0, 'unit': 'bytes',
'values': [float(5)] * 19})
def test_run(self):
port = MockPort()
test = PageLoadingPerfTest(port, "some-test", "/path/some-dir/some-test")
driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test)
output_capture = OutputCapture()
output_capture.capture_output()
try:
metrics = test._run_with_driver(driver, None)
finally:
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
self.assertEqual(actual_stdout, "")
self.assertEqual(actual_stderr, "")
self.assertEqual(actual_logs, "")
self.assertEqual(len(metrics), 1)
self.assertEqual(metrics[0].metric(), "Time")
self.assertEqual(
metrics[0].to_dict(),
{
"max": 20000,
"avg": 11000.0,
"median": 11000,
"stdev": 5627.314338711378,
"min": 2000,
"unit": "ms",
"values": [float(i * 1000) for i in range(2, 21)],
},
)
def test_run(self):
port = MockPort()
test = PageLoadingPerfTest(port, 'some-test',
'/path/some-dir/some-test')
driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test)
output_capture = OutputCapture()
output_capture.capture_output()
try:
self.assertEqual(
test.run(driver, None), {
'some-test': {
'max': 20000,
'avg': 11000.0,
'median': 11000,
'stdev': 5627.314338711378,
'min': 2000,
'unit': 'ms',
'values': [i * 1000 for i in range(2, 21)]
}
})
finally:
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output(
)
self.assertEqual(actual_stdout, '')
self.assertEqual(actual_stderr, '')
self.assertEqual(
actual_logs,
'RESULT some-test= 11000.0 ms\nmedian= 11000 ms, stdev= 5627.31433871 ms, min= 2000 ms, max= 20000 ms\n'
)
def test_run_with_bad_output(self):
printer = MockPrinter()
buildbot_output = StringIO.StringIO()
test = PageLoadingPerfTest('some-test', 'some-dir/some-test', '/path/some-dir/some-test')
driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 'some error', 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20])
self.assertEqual(test.run(driver, None, printer, buildbot_output), None)
self.assertEqual(printer.written_lines, ['error: some-test\nsome error'])
def test_run(self):
test = PageLoadingPerfTest('some-test', '/path/some-dir/some-test')
driver = TestPageLoadingPerfTest.MockDriver([
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
20
])
output_capture = OutputCapture()
output_capture.capture_output()
try:
self.assertEqual(
test.run(driver, None), {
'some-test': {
'max': 20000,
'avg': 11000.0,
'median': 11000,
'stdev': math.sqrt(570 * 1000 * 1000),
'min': 2000,
'unit': 'ms'
}
})
finally:
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output(
)
self.assertEqual(actual_stdout, '')
self.assertEqual(actual_stderr, '')
self.assertEqual(
actual_logs,
'RESULT some-test= 11000.0 ms\nmedian= 11000 ms, stdev= 23874.6727726 ms, min= 2000 ms, max= 20000 ms\n'
)
def test_run_with_memory_output(self):
port = MockPort()
test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test')
memory_results = {'Malloc': 10, 'JSHeap': 5}
self.maxDiff = None
driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test, memory_results)
output_capture = OutputCapture()
output_capture.capture_output()
try:
metrics = test._run_with_driver(driver, None)
finally:
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
self.assertEqual(actual_stdout, '')
self.assertEqual(actual_stderr, '')
self.assertEqual(actual_logs, '')
self.assertEqual(len(metrics), 3)
self.assertEqual(metrics[0].metric(), 'Time')
self.assertEqual(metrics[0].to_dict(), {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms',
'values': [float(i * 1000) for i in range(2, 21)]})
self.assertEqual(metrics[1].metric(), 'Malloc')
self.assertEqual(metrics[1].to_dict(), {'max': 10, 'avg': 10.0, 'median': 10, 'min': 10, 'stdev': 0.0, 'unit': 'bytes',
'values': [float(10)] * 19})
self.assertEqual(metrics[2].metric(), 'JSHeap')
self.assertEqual(metrics[2].to_dict(), {'max': 5, 'avg': 5.0, 'median': 5, 'min': 5, 'stdev': 0.0, 'unit': 'bytes',
'values': [float(5)] * 19})
def test_run(self):
printer = MockPrinter()
buildbot_output = StringIO.StringIO()
test = PageLoadingPerfTest('some-test', 'some-dir/some-test', '/path/some-dir/some-test')
driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20])
self.assertEqual(test.run(driver, None, printer, buildbot_output),
{'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': math.sqrt(570 * 1000 * 1000), 'min': 2000, 'unit': 'ms'}})
self.assertEqual(printer.written_lines, [])
self.assertWritten(buildbot_output, ['RESULT some-test= 11000.0 ms\n', 'median= 11000 ms, stdev= 23874.6727726 ms, min= 2000 ms, max= 20000 ms\n'])
def test_run_with_memory_output(self):
port = MockPort()
test = PageLoadingPerfTest(port, 'some-test',
'/path/some-dir/some-test')
memory_results = {'Malloc': 10, 'JSHeap': 5}
self.maxDiff = None
driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test,
memory_results)
output_capture = OutputCapture()
output_capture.capture_output()
try:
self.assertEqual(
test.run(driver, None), {
'some-test': {
'max': 20000,
'avg': 11000.0,
'median': 11000,
'stdev': 5627.314338711378,
'min': 2000,
'unit': 'ms',
'values': [i * 1000 for i in range(2, 21)]
},
'some-test:Malloc': {
'max': 10,
'avg': 10.0,
'median': 10,
'min': 10,
'stdev': 0.0,
'unit': 'bytes',
'values': [10] * 19
},
'some-test:JSHeap': {
'max': 5,
'avg': 5.0,
'median': 5,
'min': 5,
'stdev': 0.0,
'unit': 'bytes',
'values': [5] * 19
}
})
finally:
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output(
)
self.assertEqual(actual_stdout, '')
self.assertEqual(actual_stderr, '')
self.assertEqual(
actual_logs,
'RESULT some-test= 11000.0 ms\nmedian= 11000 ms, stdev= 5627.31433871 ms, min= 2000 ms, max= 20000 ms\n'
+
'RESULT some-test: Malloc= 10.0 bytes\nmedian= 10 bytes, stdev= 0.0 bytes, min= 10 bytes, max= 10 bytes\n'
+
'RESULT some-test: JSHeap= 5.0 bytes\nmedian= 5 bytes, stdev= 0.0 bytes, min= 5 bytes, max= 5 bytes\n'
)
def test_run_with_bad_output(self):
output_capture = OutputCapture()
output_capture.capture_output()
try:
test = PageLoadingPerfTest('some-test', '/path/some-dir/some-test')
driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 'some error', 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20])
self.assertEqual(test.run(driver, None), None)
finally:
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
self.assertEqual(actual_stdout, '')
self.assertEqual(actual_stderr, '')
self.assertEqual(actual_logs, 'error: some-test\nsome error\n')
def test_run_with_bad_output(self):
output_capture = OutputCapture()
output_capture.capture_output()
try:
test = PageLoadingPerfTest(None, 'some-test', '/path/some-dir/some-test')
driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 'some error', 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20])
self.assertEqual(test.run(driver, None), None)
finally:
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
self.assertEqual(actual_stdout, '')
self.assertEqual(actual_stderr, '')
self.assertEqual(actual_logs, 'error: some-test\nsome error\n')
def test_run(self):
test = PageLoadingPerfTest('some-test', '/path/some-dir/some-test')
driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20])
output_capture = OutputCapture()
output_capture.capture_output()
try:
self.assertEqual(test.run(driver, None),
{'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': math.sqrt(570 * 1000 * 1000), 'min': 2000, 'unit': 'ms'}})
finally:
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
self.assertEqual(actual_stdout, '')
self.assertEqual(actual_stderr, '')
self.assertEqual(actual_logs, 'RESULT some-test= 11000.0 ms\nmedian= 11000 ms, stdev= 23874.6727726 ms, min= 2000 ms, max= 20000 ms\n')
def test_run(self):
test = PageLoadingPerfTest(None, 'some-test', '/path/some-dir/some-test')
driver = TestPageLoadingPerfTest.MockDriver(range(1, 21))
output_capture = OutputCapture()
output_capture.capture_output()
try:
self.assertEqual(test.run(driver, None),
{'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': math.sqrt(570 * 1000 * 1000), 'min': 2000, 'unit': 'ms',
'values': [i * 1000 for i in range(2, 21)]}})
finally:
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
self.assertEqual(actual_stdout, '')
self.assertEqual(actual_stderr, '')
self.assertEqual(actual_logs, 'RESULT some-test= 11000.0 ms\nmedian= 11000 ms, stdev= 23874.6727726 ms, min= 2000 ms, max= 20000 ms\n')
def test_run_with_memory_output(self):
port = MockPort()
test = PageLoadingPerfTest(port, "some-test", "/path/some-dir/some-test")
memory_results = {"Malloc": 10, "JSHeap": 5}
self.maxDiff = None
driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test, memory_results)
output_capture = OutputCapture()
output_capture.capture_output()
try:
metrics = test._run_with_driver(driver, None)
finally:
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
self.assertEqual(actual_stdout, "")
self.assertEqual(actual_stderr, "")
self.assertEqual(actual_logs, "")
self.assertEqual(len(metrics), 3)
self.assertEqual(metrics[0].metric(), "Time")
self.assertEqual(
metrics[0].to_dict(),
{
"max": 20000,
"avg": 11000.0,
"median": 11000,
"stdev": 5627.314338711378,
"min": 2000,
"unit": "ms",
"values": [float(i * 1000) for i in range(2, 21)],
},
)
self.assertEqual(metrics[1].metric(), "Malloc")
self.assertEqual(
metrics[1].to_dict(),
{
"max": 10,
"avg": 10.0,
"median": 10,
"min": 10,
"stdev": 0.0,
"unit": "bytes",
"values": [float(10)] * 19,
},
)
self.assertEqual(metrics[2].metric(), "JSHeap")
self.assertEqual(
metrics[2].to_dict(),
{"max": 5, "avg": 5.0, "median": 5, "min": 5, "stdev": 0.0, "unit": "bytes", "values": [float(5)] * 19},
)
def test_run(self):
port = MockPort()
test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test')
driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test)
output_capture = OutputCapture()
output_capture.capture_output()
try:
self.assertEqual(test._run_with_driver(driver, None),
{'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms',
'values': [float(i * 1000) for i in range(2, 21)]}})
finally:
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
self.assertEqual(actual_stdout, '')
self.assertEqual(actual_stderr, '')
self.assertEqual(actual_logs, 'RESULT some-test= 11000 ms\nmedian= 11000 ms, stdev= 5627.31433871 ms, min= 2000 ms, max= 20000 ms\n')
def test_run_with_bad_output(self):
output_capture = OutputCapture()
output_capture.capture_output()
try:
port = MockPort()
test = PageLoadingPerfTest(port, "some-test", "/path/some-dir/some-test")
driver = TestPageLoadingPerfTest.MockDriver(
[1, 2, 3, 4, 5, 6, 7, "some error", 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], test
)
self.assertIsNone(test._run_with_driver(driver, None))
finally:
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
self.assertEqual(actual_stdout, "")
self.assertEqual(actual_stderr, "")
self.assertEqual(actual_logs, "error: some-test\nsome error\n")
def test_run(self):
port = MockPort()
test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test')
driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test)
output_capture = OutputCapture()
output_capture.capture_output()
try:
metrics = test._run_with_driver(driver, None)
finally:
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
self.assertEqual(actual_stdout, '')
self.assertEqual(actual_stderr, '')
self.assertEqual(actual_logs, '')
self.assertEqual(len(metrics), 1)
self.assertEqual(metrics[0].metric(), 'Time')
self.assertEqual(metrics[0].to_dict(), {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms',
'values': [float(i * 1000) for i in range(2, 21)]})
def test_run(self):
port = MockPort()
test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test')
driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test)
output_capture = OutputCapture()
output_capture.capture_output()
try:
metrics = test._run_with_driver(driver, None)
finally:
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
self.assertEqual(actual_stdout, '')
self.assertEqual(actual_stderr, '')
self.assertEqual(actual_logs, '')
self.assertEqual(len(metrics), 1)
self.assertEqual(metrics[0].metric(), 'Time')
self.assertEqual(metrics[0].to_dict(), {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms',
'values': [float(i * 1000) for i in range(2, 21)]})
def test_run_with_memory_output(self):
port = MockPort()
test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test')
memory_results = {'Malloc': 10, 'JSHeap': 5}
self.maxDiff = None
driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test, memory_results)
output_capture = OutputCapture()
output_capture.capture_output()
try:
self.assertEqual(test.run(driver, None),
{'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms',
'values': [i * 1000 for i in range(2, 21)]},
'some-test:Malloc': {'max': 10, 'avg': 10.0, 'median': 10, 'min': 10, 'stdev': 0.0, 'unit': 'bytes',
'values': [10] * 19},
'some-test:JSHeap': {'max': 5, 'avg': 5.0, 'median': 5, 'min': 5, 'stdev': 0.0, 'unit': 'bytes',
'values': [5] * 19}})
finally:
actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
self.assertEqual(actual_stdout, '')
self.assertEqual(actual_stderr, '')
self.assertEqual(actual_logs, 'RESULT some-test= 11000.0 ms\nmedian= 11000 ms, stdev= 5627.31433871 ms, min= 2000 ms, max= 20000 ms\n'
+ 'RESULT some-test: Malloc= 10.0 bytes\nmedian= 10 bytes, stdev= 0.0 bytes, min= 10 bytes, max= 10 bytes\n'
+ 'RESULT some-test: JSHeap= 5.0 bytes\nmedian= 5 bytes, stdev= 0.0 bytes, min= 5 bytes, max= 5 bytes\n')
