def test_run_with_memory_output(self): port = MockPort() test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test') memory_results = {'Malloc': 10, 'JSHeap': 5} self.maxDiff = None driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test, memory_results) output_capture = OutputCapture() output_capture.capture_output() try: metrics = test._run_with_driver(driver, None) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() self.assertEqual(actual_stdout, '') self.assertEqual(actual_stderr, '') self.assertEqual(actual_logs, '') self.assertEqual(len(metrics), 3) self.assertEqual(metrics[0].metric(), 'Time') self.assertEqual(metrics[0].to_dict(), {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms', 'values': [float(i * 1000) for i in range(2, 21)]}) self.assertEqual(metrics[1].metric(), 'Malloc') self.assertEqual(metrics[1].to_dict(), {'max': 10, 'avg': 10.0, 'median': 10, 'min': 10, 'stdev': 0.0, 'unit': 'bytes', 'values': [float(10)] * 19}) self.assertEqual(metrics[2].metric(), 'JSHeap') self.assertEqual(metrics[2].to_dict(), {'max': 5, 'avg': 5.0, 'median': 5, 'min': 5, 'stdev': 0.0, 'unit': 'bytes', 'values': [float(5)] * 19})
def test_run(self): port = MockPort() test = PageLoadingPerfTest(port, "some-test", "/path/some-dir/some-test") driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test) output_capture = OutputCapture() output_capture.capture_output() try: metrics = test._run_with_driver(driver, None) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() self.assertEqual(actual_stdout, "") self.assertEqual(actual_stderr, "") self.assertEqual(actual_logs, "") self.assertEqual(len(metrics), 1) self.assertEqual(metrics[0].metric(), "Time") self.assertEqual( metrics[0].to_dict(), { "max": 20000, "avg": 11000.0, "median": 11000, "stdev": 5627.314338711378, "min": 2000, "unit": "ms", "values": [float(i * 1000) for i in range(2, 21)], }, )
def test_run(self): port = MockPort() test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test') driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test) output_capture = OutputCapture() output_capture.capture_output() try: self.assertEqual( test.run(driver, None), { 'some-test': { 'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms', 'values': [i * 1000 for i in range(2, 21)] } }) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output( ) self.assertEqual(actual_stdout, '') self.assertEqual(actual_stderr, '') self.assertEqual( actual_logs, 'RESULT some-test= 11000.0 ms\nmedian= 11000 ms, stdev= 5627.31433871 ms, min= 2000 ms, max= 20000 ms\n' )
def test_run_with_bad_output(self): printer = MockPrinter() buildbot_output = StringIO.StringIO() test = PageLoadingPerfTest('some-test', 'some-dir/some-test', '/path/some-dir/some-test') driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 'some error', 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]) self.assertEqual(test.run(driver, None, printer, buildbot_output), None) self.assertEqual(printer.written_lines, ['error: some-test\nsome error'])
def test_run(self): test = PageLoadingPerfTest('some-test', '/path/some-dir/some-test') driver = TestPageLoadingPerfTest.MockDriver([ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 ]) output_capture = OutputCapture() output_capture.capture_output() try: self.assertEqual( test.run(driver, None), { 'some-test': { 'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': math.sqrt(570 * 1000 * 1000), 'min': 2000, 'unit': 'ms' } }) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output( ) self.assertEqual(actual_stdout, '') self.assertEqual(actual_stderr, '') self.assertEqual( actual_logs, 'RESULT some-test= 11000.0 ms\nmedian= 11000 ms, stdev= 23874.6727726 ms, min= 2000 ms, max= 20000 ms\n' )
def test_run(self): printer = MockPrinter() buildbot_output = StringIO.StringIO() test = PageLoadingPerfTest('some-test', 'some-dir/some-test', '/path/some-dir/some-test') driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]) self.assertEqual(test.run(driver, None, printer, buildbot_output), {'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': math.sqrt(570 * 1000 * 1000), 'min': 2000, 'unit': 'ms'}}) self.assertEqual(printer.written_lines, []) self.assertWritten(buildbot_output, ['RESULT some-test= 11000.0 ms\n', 'median= 11000 ms, stdev= 23874.6727726 ms, min= 2000 ms, max= 20000 ms\n'])
def test_run_with_memory_output(self): port = MockPort() test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test') memory_results = {'Malloc': 10, 'JSHeap': 5} self.maxDiff = None driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test, memory_results) output_capture = OutputCapture() output_capture.capture_output() try: self.assertEqual( test.run(driver, None), { 'some-test': { 'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms', 'values': [i * 1000 for i in range(2, 21)] }, 'some-test:Malloc': { 'max': 10, 'avg': 10.0, 'median': 10, 'min': 10, 'stdev': 0.0, 'unit': 'bytes', 'values': [10] * 19 }, 'some-test:JSHeap': { 'max': 5, 'avg': 5.0, 'median': 5, 'min': 5, 'stdev': 0.0, 'unit': 'bytes', 'values': [5] * 19 } }) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output( ) self.assertEqual(actual_stdout, '') self.assertEqual(actual_stderr, '') self.assertEqual( actual_logs, 'RESULT some-test= 11000.0 ms\nmedian= 11000 ms, stdev= 5627.31433871 ms, min= 2000 ms, max= 20000 ms\n' + 'RESULT some-test: Malloc= 10.0 bytes\nmedian= 10 bytes, stdev= 0.0 bytes, min= 10 bytes, max= 10 bytes\n' + 'RESULT some-test: JSHeap= 5.0 bytes\nmedian= 5 bytes, stdev= 0.0 bytes, min= 5 bytes, max= 5 bytes\n' )
def test_run_with_bad_output(self): output_capture = OutputCapture() output_capture.capture_output() try: test = PageLoadingPerfTest('some-test', '/path/some-dir/some-test') driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 'some error', 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]) self.assertEqual(test.run(driver, None), None) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() self.assertEqual(actual_stdout, '') self.assertEqual(actual_stderr, '') self.assertEqual(actual_logs, 'error: some-test\nsome error\n')
def test_run_with_bad_output(self): output_capture = OutputCapture() output_capture.capture_output() try: test = PageLoadingPerfTest(None, 'some-test', '/path/some-dir/some-test') driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 'some error', 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]) self.assertEqual(test.run(driver, None), None) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() self.assertEqual(actual_stdout, '') self.assertEqual(actual_stderr, '') self.assertEqual(actual_logs, 'error: some-test\nsome error\n')
def test_run(self): test = PageLoadingPerfTest('some-test', '/path/some-dir/some-test') driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]) output_capture = OutputCapture() output_capture.capture_output() try: self.assertEqual(test.run(driver, None), {'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': math.sqrt(570 * 1000 * 1000), 'min': 2000, 'unit': 'ms'}}) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() self.assertEqual(actual_stdout, '') self.assertEqual(actual_stderr, '') self.assertEqual(actual_logs, 'RESULT some-test= 11000.0 ms\nmedian= 11000 ms, stdev= 23874.6727726 ms, min= 2000 ms, max= 20000 ms\n')
def test_run(self): test = PageLoadingPerfTest(None, 'some-test', '/path/some-dir/some-test') driver = TestPageLoadingPerfTest.MockDriver(range(1, 21)) output_capture = OutputCapture() output_capture.capture_output() try: self.assertEqual(test.run(driver, None), {'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': math.sqrt(570 * 1000 * 1000), 'min': 2000, 'unit': 'ms', 'values': [i * 1000 for i in range(2, 21)]}}) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() self.assertEqual(actual_stdout, '') self.assertEqual(actual_stderr, '') self.assertEqual(actual_logs, 'RESULT some-test= 11000.0 ms\nmedian= 11000 ms, stdev= 23874.6727726 ms, min= 2000 ms, max= 20000 ms\n')
def test_run_with_memory_output(self): port = MockPort() test = PageLoadingPerfTest(port, "some-test", "/path/some-dir/some-test") memory_results = {"Malloc": 10, "JSHeap": 5} self.maxDiff = None driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test, memory_results) output_capture = OutputCapture() output_capture.capture_output() try: metrics = test._run_with_driver(driver, None) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() self.assertEqual(actual_stdout, "") self.assertEqual(actual_stderr, "") self.assertEqual(actual_logs, "") self.assertEqual(len(metrics), 3) self.assertEqual(metrics[0].metric(), "Time") self.assertEqual( metrics[0].to_dict(), { "max": 20000, "avg": 11000.0, "median": 11000, "stdev": 5627.314338711378, "min": 2000, "unit": "ms", "values": [float(i * 1000) for i in range(2, 21)], }, ) self.assertEqual(metrics[1].metric(), "Malloc") self.assertEqual( metrics[1].to_dict(), { "max": 10, "avg": 10.0, "median": 10, "min": 10, "stdev": 0.0, "unit": "bytes", "values": [float(10)] * 19, }, ) self.assertEqual(metrics[2].metric(), "JSHeap") self.assertEqual( metrics[2].to_dict(), {"max": 5, "avg": 5.0, "median": 5, "min": 5, "stdev": 0.0, "unit": "bytes", "values": [float(5)] * 19}, )
def test_run(self): port = MockPort() test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test') driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test) output_capture = OutputCapture() output_capture.capture_output() try: self.assertEqual(test._run_with_driver(driver, None), {'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms', 'values': [float(i * 1000) for i in range(2, 21)]}}) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() self.assertEqual(actual_stdout, '') self.assertEqual(actual_stderr, '') self.assertEqual(actual_logs, 'RESULT some-test= 11000 ms\nmedian= 11000 ms, stdev= 5627.31433871 ms, min= 2000 ms, max= 20000 ms\n')
def test_run_with_bad_output(self): output_capture = OutputCapture() output_capture.capture_output() try: port = MockPort() test = PageLoadingPerfTest(port, "some-test", "/path/some-dir/some-test") driver = TestPageLoadingPerfTest.MockDriver( [1, 2, 3, 4, 5, 6, 7, "some error", 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], test ) self.assertIsNone(test._run_with_driver(driver, None)) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() self.assertEqual(actual_stdout, "") self.assertEqual(actual_stderr, "") self.assertEqual(actual_logs, "error: some-test\nsome error\n")
def test_run(self): port = MockPort() test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test') driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test) output_capture = OutputCapture() output_capture.capture_output() try: metrics = test._run_with_driver(driver, None) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() self.assertEqual(actual_stdout, '') self.assertEqual(actual_stderr, '') self.assertEqual(actual_logs, '') self.assertEqual(len(metrics), 1) self.assertEqual(metrics[0].metric(), 'Time') self.assertEqual(metrics[0].to_dict(), {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms', 'values': [float(i * 1000) for i in range(2, 21)]})
def test_run_with_memory_output(self): port = MockPort() test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test') memory_results = {'Malloc': 10, 'JSHeap': 5} self.maxDiff = None driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test, memory_results) output_capture = OutputCapture() output_capture.capture_output() try: self.assertEqual(test.run(driver, None), {'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms', 'values': [i * 1000 for i in range(2, 21)]}, 'some-test:Malloc': {'max': 10, 'avg': 10.0, 'median': 10, 'min': 10, 'stdev': 0.0, 'unit': 'bytes', 'values': [10] * 19}, 'some-test:JSHeap': {'max': 5, 'avg': 5.0, 'median': 5, 'min': 5, 'stdev': 0.0, 'unit': 'bytes', 'values': [5] * 19}}) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() self.assertEqual(actual_stdout, '') self.assertEqual(actual_stderr, '') self.assertEqual(actual_logs, 'RESULT some-test= 11000.0 ms\nmedian= 11000 ms, stdev= 5627.31433871 ms, min= 2000 ms, max= 20000 ms\n' + 'RESULT some-test: Malloc= 10.0 bytes\nmedian= 10 bytes, stdev= 0.0 bytes, min= 10 bytes, max= 10 bytes\n' + 'RESULT some-test: JSHeap= 5.0 bytes\nmedian= 5 bytes, stdev= 0.0 bytes, min= 5 bytes, max= 5 bytes\n')