def testParseComplexMetricName(self):
full_metric_name = ('memory:chrome:all_processes:reported_by_os:'
'system_memory:native_heap:'
'proportional_resident_size_avg/blank_about/'
'blank_about_blank')
chart_name = (
'blank_about@@memory:chrome:all_processes:reported_by_os:'
'system_memory:native_heap:proportional_resident_size_avg')
trace_name = 'blank:about:blank'
metric = chart_name, trace_name
keys = 'blank', 'about'
lower_values = ','.join(
self.MakeCharts(metric=metric,
seed='lower',
mu=10,
sigma=1,
n=10,
keys=keys))
higher_values = ','.join(
self.MakeCharts(metric=metric,
seed='higher',
mu=20,
sigma=2,
n=10,
keys=keys))
result = compare_samples.CompareSamples(lower_values, higher_values,
full_metric_name).stdout
print result
result = json.loads(result)
self.assertEqual(result['result']['significance'], REJECT)
def testCompareMissingFile(self):
metric = ('some_chart', 'some_trace')
lower_values = ','.join(
[self.MakeChart(metric=metric, seed='lower', mu=10, sigma=1, n=5)])
higher_values = '/path/does/not/exist.json'
with self.assertRaises(RuntimeError):
compare_samples.CompareSamples(lower_values, higher_values,
'/'.join(metric))
def testCompareBadChart(self):
metric = ('some_chart', 'some_trace')
lower_values = ','.join([self.MakeChart(metric=metric, seed='lower',
mu=10, sigma=1, n=5)])
higher_values = self.NewJsonTempfile(['obviously', 'not', 'a', 'chart]'])
result = json.loads(compare_samples.CompareSamples(
lower_values, higher_values, '/'.join(metric)).stdout)
self.assertEqual(result['result']['significance'], NEED_MORE_DATA)
def testCompareChartJsonHistogram(self):
metric = ('some_chart', 'some_trace')
lower_values = ','.join(self.MakeMultipleChartJSONHistograms(
metric=metric, seed='lower', mu=10, sigma=1, n=100, m=10))
higher_values = ','.join(self.MakeMultipleChartJSONHistograms(
metric=metric, seed='higher', mu=20, sigma=2, n=100, m=10))
result = json.loads(compare_samples.CompareSamples(
lower_values, higher_values, '/'.join(metric)).stdout)
self.assertEqual(result['result']['significance'], REJECT)
def testCompareInsufficientData(self):
metric = ('some_chart', 'some_trace')
lower_values = ','.join([self.MakeChart(metric=metric, seed='lower',
mu=10, sigma=1, n=5)])
higher_values = ','.join([self.MakeChart(metric=metric, seed='higher',
mu=10.40, sigma=0.95, n=5)])
result = json.loads(compare_samples.CompareSamples(
lower_values, higher_values, '/'.join(metric)).stdout)
self.assertEqual(result['result']['significance'], NEED_MORE_DATA)
def testCompareClearRegressionListOfScalars(self):
metric = ('some_chart', 'some_trace')
lower_values = ','.join(self.MakeCharts(metric=metric, seed='lower',
mu=10, sigma=1, n=10))
higher_values = ','.join(self.MakeCharts(metric=metric, seed='higher',
mu=20, sigma=2, n=10))
result = json.loads(compare_samples.CompareSamples(
lower_values, higher_values, '/'.join(metric)).stdout)
self.assertEqual(result['result']['significance'], REJECT)
def testCompareBadChart(self):
metric = ('some_chart', 'some_trace')
lower_values = ','.join(
[self.MakeChart(metric=metric, seed='lower', mu=10, sigma=1, n=5)])
higher_values = self.NewJsonTempfile(
['obviously', 'not', 'a', 'chart]'])
with self.assertRaises(RuntimeError):
compare_samples.CompareSamples(lower_values, higher_values,
'/'.join(metric))
def testCompareBuildbotOutput(self):
bb = os.path.join(os.path.dirname(__file__),
'buildbot_output_for_compare_samples_test.txt')
result = compare_samples.CompareSamples(
bb, bb, 'DrawCallPerf_gl/score',
data_format='buildbot')
result = json.loads(result.stdout)
self.assertEqual(result['result']['significance'], NEED_MORE_DATA)
self.assertEqual(Mean(result['sampleA']), 4123)
self.assertEqual(Mean(result['sampleB']), 4123)
def testCompareTIRLabelMissingSummary(self):
tir_metric = ('some_chart', 'some_label')
tir_metric_name = ('%s@@%s' % (tir_metric[1], tir_metric[0]), 'summary')
lower_values = ','.join(self.MakeCharts(
metric=tir_metric_name, seed='lower', mu=10, sigma=1, n=10))
higher_values = ','.join(self.MakeCharts(
metric=tir_metric_name, seed='higher', mu=20, sigma=2, n=10))
result = json.loads(compare_samples.CompareSamples(
lower_values, higher_values, '/'.join(tir_metric)).stdout)
self.assertEqual(result['result']['significance'], REJECT)
def testCompareValuesets(self):
vs = os.path.join(os.path.dirname(__file__),
'valueset_output_for_compare_samples_test.json')
result = compare_samples.CompareSamples(
vs, vs, 'timeToFirstContentfulPaint/pcv1-cold/'
'http___www.rambler.ru_', data_format='valueset')
result = json.loads(result.stdout)
self.assertEqual(result['result']['significance'], NEED_MORE_DATA)
self.assertAlmostEqual(Mean(result['sampleA']), 75.3177999958396)
self.assertAlmostEqual(Mean(result['sampleB']), 75.3177999958396)
def testCompareUnlikelyRegressionWithMultipleRuns(self):
metric = ('some_chart', 'some_trace')
lower_values = ','.join(
self.MakeCharts(
metric=metric, seed='lower', mu=10, sigma=1, n=20))
higher_values = ','.join(
self.MakeCharts(
metric=metric, seed='higher', mu=10.01, sigma=0.95, n=20))
result = json.loads(compare_samples.CompareSamples(
lower_values, higher_values, '/'.join(metric)).stdout)
self.assertEqual(result['result']['significance'], FAIL_TO_REJECT)
def testCompareMissingMetric(self):
metric = ('some_chart', 'some_trace')
lower_values = ','.join(
[self.MakeChart(metric=metric, seed='lower', mu=10, sigma=1, n=5)])
higher_values = ','.join([
self.MakeChart(metric=metric, seed='higher', mu=20, sigma=2, n=5)
])
metric = ('some_chart', 'missing_trace')
with self.assertRaises(RuntimeError):
compare_samples.CompareSamples(lower_values, higher_values,
'/'.join(metric))
def testCompareGroupingLabel(self):
parts = ('some_chart', 'some_label', 'some_trace')
metric_name = ('%s@@%s' % (parts[1], parts[0]), parts[2])
lower_values = ','.join(
self.MakeCharts(metric=metric_name,
seed='lower',
mu=10,
sigma=1,
n=10))
higher_values = ','.join(
self.MakeCharts(metric=metric_name,
seed='higher',
mu=20,
sigma=2,
n=10))
result = json.loads(
compare_samples.CompareSamples(lower_values, higher_values,
'/'.join(parts)).stdout)
self.assertEqual(result['result']['significance'], REJECT)