def testParseComplexMetricName(self): full_metric_name = ('memory:chrome:all_processes:reported_by_os:' 'system_memory:native_heap:' 'proportional_resident_size_avg/blank_about/' 'blank_about_blank') chart_name = ( 'blank_about@@memory:chrome:all_processes:reported_by_os:' 'system_memory:native_heap:proportional_resident_size_avg') trace_name = 'blank:about:blank' metric = chart_name, trace_name keys = 'blank', 'about' lower_values = ','.join( self.MakeCharts(metric=metric, seed='lower', mu=10, sigma=1, n=10, keys=keys)) higher_values = ','.join( self.MakeCharts(metric=metric, seed='higher', mu=20, sigma=2, n=10, keys=keys)) result = compare_samples.CompareSamples(lower_values, higher_values, full_metric_name).stdout print result result = json.loads(result) self.assertEqual(result['result']['significance'], REJECT)
def testCompareMissingFile(self): metric = ('some_chart', 'some_trace') lower_values = ','.join( [self.MakeChart(metric=metric, seed='lower', mu=10, sigma=1, n=5)]) higher_values = '/path/does/not/exist.json' with self.assertRaises(RuntimeError): compare_samples.CompareSamples(lower_values, higher_values, '/'.join(metric))
def testCompareBadChart(self): metric = ('some_chart', 'some_trace') lower_values = ','.join([self.MakeChart(metric=metric, seed='lower', mu=10, sigma=1, n=5)]) higher_values = self.NewJsonTempfile(['obviously', 'not', 'a', 'chart]']) result = json.loads(compare_samples.CompareSamples( lower_values, higher_values, '/'.join(metric)).stdout) self.assertEqual(result['result']['significance'], NEED_MORE_DATA)
def testCompareChartJsonHistogram(self): metric = ('some_chart', 'some_trace') lower_values = ','.join(self.MakeMultipleChartJSONHistograms( metric=metric, seed='lower', mu=10, sigma=1, n=100, m=10)) higher_values = ','.join(self.MakeMultipleChartJSONHistograms( metric=metric, seed='higher', mu=20, sigma=2, n=100, m=10)) result = json.loads(compare_samples.CompareSamples( lower_values, higher_values, '/'.join(metric)).stdout) self.assertEqual(result['result']['significance'], REJECT)
def testCompareInsufficientData(self): metric = ('some_chart', 'some_trace') lower_values = ','.join([self.MakeChart(metric=metric, seed='lower', mu=10, sigma=1, n=5)]) higher_values = ','.join([self.MakeChart(metric=metric, seed='higher', mu=10.40, sigma=0.95, n=5)]) result = json.loads(compare_samples.CompareSamples( lower_values, higher_values, '/'.join(metric)).stdout) self.assertEqual(result['result']['significance'], NEED_MORE_DATA)
def testCompareClearRegressionListOfScalars(self): metric = ('some_chart', 'some_trace') lower_values = ','.join(self.MakeCharts(metric=metric, seed='lower', mu=10, sigma=1, n=10)) higher_values = ','.join(self.MakeCharts(metric=metric, seed='higher', mu=20, sigma=2, n=10)) result = json.loads(compare_samples.CompareSamples( lower_values, higher_values, '/'.join(metric)).stdout) self.assertEqual(result['result']['significance'], REJECT)
def testCompareBadChart(self): metric = ('some_chart', 'some_trace') lower_values = ','.join( [self.MakeChart(metric=metric, seed='lower', mu=10, sigma=1, n=5)]) higher_values = self.NewJsonTempfile( ['obviously', 'not', 'a', 'chart]']) with self.assertRaises(RuntimeError): compare_samples.CompareSamples(lower_values, higher_values, '/'.join(metric))
def testCompareBuildbotOutput(self): bb = os.path.join(os.path.dirname(__file__), 'buildbot_output_for_compare_samples_test.txt') result = compare_samples.CompareSamples( bb, bb, 'DrawCallPerf_gl/score', data_format='buildbot') result = json.loads(result.stdout) self.assertEqual(result['result']['significance'], NEED_MORE_DATA) self.assertEqual(Mean(result['sampleA']), 4123) self.assertEqual(Mean(result['sampleB']), 4123)
def testCompareTIRLabelMissingSummary(self): tir_metric = ('some_chart', 'some_label') tir_metric_name = ('%s@@%s' % (tir_metric[1], tir_metric[0]), 'summary') lower_values = ','.join(self.MakeCharts( metric=tir_metric_name, seed='lower', mu=10, sigma=1, n=10)) higher_values = ','.join(self.MakeCharts( metric=tir_metric_name, seed='higher', mu=20, sigma=2, n=10)) result = json.loads(compare_samples.CompareSamples( lower_values, higher_values, '/'.join(tir_metric)).stdout) self.assertEqual(result['result']['significance'], REJECT)
def testCompareValuesets(self): vs = os.path.join(os.path.dirname(__file__), 'valueset_output_for_compare_samples_test.json') result = compare_samples.CompareSamples( vs, vs, 'timeToFirstContentfulPaint/pcv1-cold/' 'http___www.rambler.ru_', data_format='valueset') result = json.loads(result.stdout) self.assertEqual(result['result']['significance'], NEED_MORE_DATA) self.assertAlmostEqual(Mean(result['sampleA']), 75.3177999958396) self.assertAlmostEqual(Mean(result['sampleB']), 75.3177999958396)
def testCompareUnlikelyRegressionWithMultipleRuns(self): metric = ('some_chart', 'some_trace') lower_values = ','.join( self.MakeCharts( metric=metric, seed='lower', mu=10, sigma=1, n=20)) higher_values = ','.join( self.MakeCharts( metric=metric, seed='higher', mu=10.01, sigma=0.95, n=20)) result = json.loads(compare_samples.CompareSamples( lower_values, higher_values, '/'.join(metric)).stdout) self.assertEqual(result['result']['significance'], FAIL_TO_REJECT)
def testCompareMissingMetric(self): metric = ('some_chart', 'some_trace') lower_values = ','.join( [self.MakeChart(metric=metric, seed='lower', mu=10, sigma=1, n=5)]) higher_values = ','.join([ self.MakeChart(metric=metric, seed='higher', mu=20, sigma=2, n=5) ]) metric = ('some_chart', 'missing_trace') with self.assertRaises(RuntimeError): compare_samples.CompareSamples(lower_values, higher_values, '/'.join(metric))
def testCompareGroupingLabel(self): parts = ('some_chart', 'some_label', 'some_trace') metric_name = ('%s@@%s' % (parts[1], parts[0]), parts[2]) lower_values = ','.join( self.MakeCharts(metric=metric_name, seed='lower', mu=10, sigma=1, n=10)) higher_values = ','.join( self.MakeCharts(metric=metric_name, seed='higher', mu=20, sigma=2, n=10)) result = json.loads( compare_samples.CompareSamples(lower_values, higher_values, '/'.join(parts)).stdout) self.assertEqual(result['result']['significance'], REJECT)